diff --git a/src/agent_env/artifact/artifact.py b/src/agent_env/artifact/artifact.py index 2e817a35..b51ee8e4 100644 --- a/src/agent_env/artifact/artifact.py +++ b/src/agent_env/artifact/artifact.py @@ -2,6 +2,9 @@ from __future__ import annotations +from collections.abc import Mapping +from dataclasses import dataclass +from pathlib import Path from typing import TYPE_CHECKING, Any, ClassVar, Optional, Self from pydantic import BaseModel, Field, field_validator @@ -21,6 +24,23 @@ def _write_twin(data: dict[str, Any], legacy: str, neutral: str, value: Any) -> data[legacy] = data[neutral] = value +@dataclass(frozen=True) +class FolderChild: + """An artifact written from a bundle folder besides the one the folder is for.""" + + id: str + path: str # where in the folder it's written from; "" for the folder itself + environment_name: str | None = None # the environment it seeds, for one that seeds an env's server + + +@dataclass(frozen=True) +class ArtifactFolder: + """What writing an artifact from its bundle folder reads there, and writes besides the artifact itself.""" + + files: Mapping[str, Path] # every file its write reads, keyed by its path in the folder; the ledger hashes them + children: tuple[FolderChild, ...] = () + + class Artifact(BaseModel): """Base class for all environment artifacts. Immutable - each modification creates a new version.""" @@ -29,6 +49,7 @@ class Artifact(BaseModel): type: str = Field(description="The type of Artifact") toml_refs: ClassVar[tuple[EntityRef, ...]] = () + toml_keys: ClassVar[dict[str, type]] = {} # what an artifact.toml may set, for a type without an accept_toml @field_validator("type", mode="before") @classmethod @@ -74,6 +95,14 @@ def from_toml(cls, data: dict[str, Any], ctx: AuthoringContext) -> Self: fields = {key: value for key, value in data.items() if key not in ("id", "type", "version")} return cls.put(id=ctx.id, **fields) + @classmethod + def bundle_folder(cls, ctx: AuthoringContext) -> ArtifactFolder | None: + """What ``from_toml`` reads from the entry's folder and writes besides ``ctx.id``, which a bundle checks and + its ledger hashes before anything is written; None for a type a bundle doesn't write from a folder. A type + is written from one only when the class defining its ``from_toml`` defines this too. Raises BundleError + (``ctx.refuse``) listing what's wrong with the folder.""" + return None + @classmethod def query(cls) -> "ArtifactQuery": from agent_env.artifact.store import ArtifactQuery, get_artifact_store diff --git a/src/agent_env/artifact/artifacts/cli.py b/src/agent_env/artifact/artifacts/cli.py index 04df98d2..cfa51c91 100644 --- a/src/agent_env/artifact/artifacts/cli.py +++ b/src/agent_env/artifact/artifacts/cli.py @@ -15,7 +15,7 @@ from pydantic import ConfigDict, Field, model_serializer -from agent_env.artifact.artifact import Artifact, _write_twin +from agent_env.artifact.artifact import Artifact, ArtifactFolder, _write_twin from agent_env.store.ids import derive_id if TYPE_CHECKING: @@ -129,6 +129,11 @@ def from_toml(cls, data: dict, ctx: AuthoringContext) -> "CliArtifact": return cls.put(ctx.id, command_name=fields["command_name"], entrypoint=fields["entrypoint"], files=ctx.files()) + @classmethod + def bundle_folder(cls, ctx: AuthoringContext) -> ArtifactFolder: + """What ``from_toml`` reads from the folder: every file it holds.""" + return ArtifactFolder(ctx.files()) + def get_cli_files(self) -> "FileArtifactUniverse": from agent_env.artifact.artifacts.file_artifact_universe import FileArtifactUniverse return FileArtifactUniverse.get(self.cli_files_id) diff --git a/src/agent_env/artifact/artifacts/environment.py b/src/agent_env/artifact/artifacts/environment.py index c8819521..144ac24e 100644 --- a/src/agent_env/artifact/artifacts/environment.py +++ b/src/agent_env/artifact/artifacts/environment.py @@ -7,7 +7,7 @@ from pydantic import ConfigDict, Field, model_serializer -from agent_env.artifact.artifact import Artifact +from agent_env.artifact.artifact import Artifact, ArtifactFolder, FolderChild from agent_env.artifact.artifacts.file import FileArtifact from agent_env.artifact.ref import ArtifactRef from agent_env.entity_refs import EntityRef @@ -95,11 +95,31 @@ def from_toml(cls, data: dict, ctx: AuthoringContext) -> EnvironmentArtifact: if "file" in fields: file = ctx.artifact(fields["file"], FileArtifact) else: - filename, path = ctx.file() + [(key, path)] = cls.bundle_folder(ctx).files.items() + filename = key.rsplit("/", 1)[-1] file = FileArtifact.put_attempt(cls.derived_file_id(ctx.id), file_path=str(path), filename=filename, description=fields.get("description", filename)) return cls.put(ctx.id, environment_name=fields["environment_name"], file_artifact=file) + @classmethod + def bundle_folder(cls, ctx: AuthoringContext) -> ArtifactFolder: + """What ``from_toml`` reads from the folder, its one file, and writes besides the environment, the file, as + ``__file``; or nothing, when ``file`` names the artifact to wrap and the folder holds only artifact.toml.""" + files = ctx.files(empty_ok=True) + names_its_file = ("file", *(stored for stored, key in cls.toml_stored_names.items() if key == "file")) + if not (isinstance(ctx.entry.config, dict) and any(name in ctx.entry.config for name in names_its_file)): + if not files: + ctx.refuse([f"has no file for the environment to wrap ({', '.join(ctx.leavings)} don't count); put " + "its one file here, or name the artifact to wrap with file"]) + files = ctx.only(files, "name the artifact to wrap with file, or give each other file an artifact folder " + "of its own") + return ArtifactFolder(files, (FolderChild(cls.derived_file_id(ctx.id), ""),)) + if files: + ctx.refuse([f"its file is the artifact file names, so the folder holds only artifact.toml, and it also has " + f"{', '.join(repr(name) for name in files)}; give the file an artifact folder of its own, or " + "leave out file to wrap the folder's one file"]) + return ArtifactFolder({}) + @classmethod def accept_toml(cls, data: dict, ctx: AuthoringContext) -> dict: """The keys of ``data``, an artifact.toml, an environment takes. Raises BundleError listing every problem.""" diff --git a/src/agent_env/artifact/artifacts/environment_universe.py b/src/agent_env/artifact/artifacts/environment_universe.py index 684603eb..c8df9745 100644 --- a/src/agent_env/artifact/artifacts/environment_universe.py +++ b/src/agent_env/artifact/artifacts/environment_universe.py @@ -8,15 +8,26 @@ from pydantic import AliasChoices, ConfigDict, Field, model_serializer +from agent_env.artifact.artifact import ArtifactFolder, FolderChild from agent_env.artifact.artifacts.environment import EnvironmentArtifact +from agent_env.artifact.artifacts.environment_universe_folder import ( + METADATA, + UniverseFolder, + UniverseFolderError, + environment_id, + metadata_id, + read_folder, +) from agent_env.artifact.artifacts.file import FileArtifact from agent_env.artifact.ref import ArtifactRef from agent_env.artifact.universe import Universe from agent_env.entity_refs import EntityRef -from agent_env.store.ids import derive_id logger = logging.getLogger(__name__) +_NEEDS_AN_ENVIRONMENT = ("a universe needs an environment: a folder of its own, named after it and holding its file, or " + "one environment_artifacts names") + if TYPE_CHECKING: from agent_env.bundle.authoring import AuthoringContext @@ -37,7 +48,7 @@ class EnvironmentUniverseArtifact(Universe): "metadata_refs": "a metadata// folder holding its file"} # What a universe's metadata is called: the folder `environment-universe get --output-dir` writes its files to, and # the segment of the ids they're written under. - metadata_name: ClassVar[str] = "metadata" + metadata_name: ClassVar[str] = METADATA type: Literal["environment_universe"] = "environment_universe" # serialization_alias pins the emitted key: the attribute is renamed, the document is not. @@ -117,12 +128,12 @@ def put( @staticmethod def derived_environment_id(id: str, environment_name: str) -> str: """The id an environment is written under when it's written with the universe: ``__``.""" - return derive_id(id, environment_name) + return environment_id(id, environment_name) - @classmethod - def derived_metadata_id(cls, id: str, key: str) -> str: + @staticmethod + def derived_metadata_id(id: str, key: str) -> str: """The id a metadata file is written under when it's written with the universe: ``__metadata__``.""" - return derive_id(derive_id(id, cls.metadata_name), key) + return metadata_id(id, key) @classmethod def from_toml(cls, data: dict, ctx: AuthoringContext) -> EnvironmentUniverseArtifact: @@ -130,15 +141,16 @@ def from_toml(cls, data: dict, ctx: AuthoringContext) -> EnvironmentUniverseArti environment artifacts' ids) and its folder under ``ctx.id``, and return it. The folder is laid out as ``environment-universe get --output-dir`` writes one: each ``/`` folder's file is written as an environment of that name, ``__`` over ``____file``, and each ``metadata//`` - folder's file as ``__metadata__``. Its environments are the folders', in name order, then the ones - ``environment_artifacts`` names. Nothing is written when it has none, or two of one name.""" + folder's file as ``__metadata__``: a name or key holding ``/`` is nested folders, and one an id can't + hold is written under a hash of it (``environment_universe_folder``). Its environments are the folders', in + name order, then the ones ``environment_artifacts`` names. Nothing is written when it has none, or two of one + name.""" fields = cls.accept_toml(data, ctx) - layout = ctx.universe() + layout = cls._read_folder(ctx) named = [ctx.artifact(ref, EnvironmentArtifact) for ref in fields.get("environment_artifacts", [])] names = Counter([laid.name for laid in layout.environments] + [env.environment_name for env in named]) if not names: - ctx.refuse([ctx.config_problem("a universe needs an environment: a folder of its own, named after it and " - "holding its file, or one environment_artifacts names")]) + ctx.refuse([ctx.config_problem(_NEEDS_AN_ENVIRONMENT)]) if repeated := sorted(name for name, count in names.items() if count > 1): ctx.refuse([ctx.config_problem(f"two of its environments are named {name!r}; a universe's environments " "need names of their own") for name in repeated]) @@ -153,6 +165,30 @@ def from_toml(cls, data: dict, ctx: AuthoringContext) -> EnvironmentUniverseArti for laid in layout.metadata} return cls.put(ctx.id, environment_artifacts=environments + named, metadata=metadata or None) + @classmethod + def bundle_folder(cls, ctx: AuthoringContext) -> ArtifactFolder: + """What ``from_toml`` reads from the folder, every file ``read_folder`` lays out, and writes besides the + universe: each environment, seeding the environment of its name, its file, and each metadata file. A folder + holding no environment needs an artifact.toml naming one.""" + layout = cls._read_folder(ctx) + if not layout.environments and not (isinstance(ctx.entry.config, dict) + and ctx.entry.config.get("environment_artifacts")): + ctx.refuse([_NEEDS_AN_ENVIRONMENT]) + environments = (child for laid in layout.environments for child in ( + FolderChild(laid.environment_id, laid.name, environment_name=laid.name), FolderChild(laid.file_id, laid.name))) + metadata = (FolderChild(laid.file_id, f"{METADATA}/{laid.name}") for laid in layout.metadata) + return ArtifactFolder(layout.files, (*environments, *metadata)) + + @classmethod + def _read_folder(cls, ctx: AuthoringContext) -> UniverseFolder: + """The universe's folder, read as ``environment-universe get --output-dir`` writes one (``read_folder``). + Raises BundleError listing every problem.""" + files, folders = ctx.listing() + try: + return read_folder(ctx.id, files, folders) + except UniverseFolderError as e: + ctx.refuse_within(e.problems) + @classmethod def accept_toml(cls, data: dict, ctx: AuthoringContext) -> dict: """The keys of ``data``, an artifact.toml, a universe takes. Raises BundleError listing every problem.""" diff --git a/src/agent_env/artifact/artifacts/environment_universe_folder.py b/src/agent_env/artifact/artifacts/environment_universe_folder.py new file mode 100644 index 00000000..5a020819 --- /dev/null +++ b/src/agent_env/artifact/artifacts/environment_universe_folder.py @@ -0,0 +1,156 @@ +"""The folder an environment universe is written to and read back from. + +``environment-universe get --output-dir`` writes each environment's file to ``/`` and each +metadata file to ``metadata//`` (``environment_path``, ``metadata_path``), a name or key holding ``/`` in +nested folders. ``read_folder`` is its exact inverse, which a bundle's environment_universe folder is written from, +and ``environment_id`` and ``metadata_id`` name what's written from each folder. +The format's own limits: an environment whose name starts with ``metadata/`` reads back as a metadata key, names are +read in NFC, and names that differ only in case are one folder on a case-insensitive disk. +""" + +from __future__ import annotations + +import functools +import hashlib +import unicodedata +from collections.abc import Callable, Iterable, Mapping +from dataclasses import dataclass +from pathlib import Path + +from agent_env.artifact.artifacts.environment import EnvironmentArtifact +from agent_env.store.ids import derive_id, validate_local_id + +# What a universe's metadata is called: the folder its files go in, and the segment of the ids they're written under. +METADATA = "metadata" + + +def environment_id(universe_id: str, environment_name: str) -> str: + """The id an environment is written under when it's written with its universe: ``__``.""" + return derive_id(universe_id, environment_name) + + +def metadata_id(universe_id: str, key: str) -> str: + """The id a metadata file is written under when it's written with its universe: ``__metadata__``.""" + return derive_id(derive_id(universe_id, METADATA), key) + + +def environment_path(folder: Path, environment_name: str, filename: str) -> Path: + """Where the file of a universe's environment goes in the universe's folder.""" + return folder / environment_name / filename + + +def metadata_path(folder: Path, key: str, filename: str) -> Path: + """Where a universe's metadata file goes in the universe's folder.""" + return folder / METADATA / key / filename + + +@dataclass(frozen=True) +class FolderEntry: + """One file of a universe's folder, and the ids it's written under.""" + + name: str # its environment's name, or its metadata key + filename: str + path: Path + file_id: str + environment_id: str | None # None for a metadata file + + +@dataclass(frozen=True) +class UniverseFolder: + """A universe's folder, read: its environments, its metadata files, and every file it holds.""" + + environments: tuple[FolderEntry, ...] + metadata: tuple[FolderEntry, ...] + files: Mapping[str, Path] + + +class UniverseFolderError(ValueError): + """Why a folder can't be read as a universe: ``problems``, each ``: ``.""" + + def __init__(self, problems: list[str]): + super().__init__("; ".join(problems)) + self.problems = problems + + +def read_folder(universe_id: str, files: Mapping[str, Path], + folders: Iterable[tuple[str, ...]]) -> UniverseFolder: + """The environments and metadata files a universe's folder holds, from its files (``{path in the folder: path}``) + and its folders (the names on each path to one): each folder directly holding one file is an environment named by + its path from the universe's folder, or, under ``metadata/``, a metadata key named by its path from there. An + environment is written as ``__`` over ``____file`` and a metadata file as + ``__metadata__``, with a hash of the name in its place where an id can't hold it. Raises + UniverseFolderError listing every problem.""" + folders = set(folders) + # A metadata/ folder spelled in another case is a typo, as for every reserved name in a bundle. + near = sorted(parts[0] for parts in folders + if len(parts) == 1 and parts[0] != METADATA and _fold(parts[0]) == _fold(METADATA)) + problems = [f"{name}: rename to {METADATA}; names are case-sensitive, and an environment named {name} goes in an " + "artifact folder of its own, named in environment_artifacts" for name in near] + # Every folder counts, so one left without its file is refused rather than dropped. + held: dict[tuple[str, ...], list[str]] = {parts: [] for parts in folders + if parts[0] not in near and parts != (METADATA,)} + for key in files: + parts = tuple(key.split("/")) + if parts[0] in near: + continue + if len(parts) == 1: + problems.append(f"{key}: a universe folder holds only artifact.toml and a folder for each environment; " + "move it into /, or give it an artifact folder of its own") + elif parts[:-1] == (METADATA,): + problems.append(f"{key}: {METADATA}/ holds a folder for each key, with that key's one file in it " + f"({METADATA}//); an environment named {METADATA} goes in an artifact folder " + "of its own, named in environment_artifacts") + else: + held.setdefault(parts[:-1], []).append(key) + environments, metadata_files = [], [] + # A folder holding only folders is part of their names. + holders = {parts[:depth] for parts in held for depth in range(1, len(parts))} + for parts, keys in sorted(held.items(), key=lambda item: "/".join(item[0])): + folder = "/".join(parts) + if not keys and parts in holders: + continue + if len(keys) != 1: + listed = f" ({', '.join(repr(key.rsplit('/', 1)[-1]) for key in keys)})" if keys else "" + problems.append(f"{folder}: holds one file, and has {len(keys)}{listed}") + continue + metadata = parts[0] == METADATA + name = "/".join(parts[1:] if metadata else parts) + if metadata: + env_id, file_id = None, _child_id(functools.partial(metadata_id, universe_id), name, environment=False) + else: + env_id = _child_id(functools.partial(environment_id, universe_id), name, environment=True) + file_id = EnvironmentArtifact.derived_file_id(env_id) + try: + for id in (env_id, file_id): + if id is not None: + validate_local_id(id) + except ValueError as e: + problems.append(f"{folder}: can't name the artifacts written from it ({e}); give the universe a shorter " + "name") + continue + entry = FolderEntry(name, keys[0].rsplit("/", 1)[-1], files[keys[0]], file_id, env_id) + (metadata_files if metadata else environments).append(entry) + if problems: + raise UniverseFolderError(problems) + return UniverseFolder(tuple(environments), tuple(metadata_files), files) + + +def _child_id(derive: Callable[[str], str], name: str, *, environment: bool) -> str: + """The id a universe's environment or metadata file named ``name`` is written under, as ``derive``, the + universe's own naming, derives it from a name: from ``name``, or, for a name holding ``__``, which could clash + with another's ids, or one an id can't hold (with an environment's file id, ``__file``), from + ``sha256__<16 hex of its hash>``, which holds ``__``, so no name's own id can be.""" + if "__" not in name: + id = derive(name) + try: + for derived in (id, EnvironmentArtifact.derived_file_id(id)) if environment else (id,): + validate_local_id(derived) + return id + except ValueError: + pass + return derive(f"sha256__{hashlib.sha256(name.encode('utf-8')).hexdigest()[:16]}") + + +def _fold(name: str) -> str: + """The caseless form names are compared in: casefold output isn't always NFC, so normalize both sides.""" + return unicodedata.normalize("NFC", unicodedata.normalize("NFC", name).casefold()) diff --git a/src/agent_env/artifact/artifacts/file.py b/src/agent_env/artifact/artifacts/file.py index e4d7a58e..6b822a32 100644 --- a/src/agent_env/artifact/artifacts/file.py +++ b/src/agent_env/artifact/artifacts/file.py @@ -8,7 +8,7 @@ from pydantic import ConfigDict, Field, model_serializer -from agent_env.artifact.artifact import Artifact, _write_twin +from agent_env.artifact.artifact import Artifact, ArtifactFolder, _write_twin from agent_env.config import get_config if TYPE_CHECKING: @@ -136,6 +136,11 @@ def from_toml(cls, data: dict, ctx: AuthoringContext) -> Self: return cls.put_attempt(ctx.id, description=fields.get("description", filename), file_path=str(path), filename=filename) + @classmethod + def bundle_folder(cls, ctx: AuthoringContext) -> ArtifactFolder: + """What ``from_toml`` reads from the folder: its one file.""" + return ArtifactFolder(ctx.only(ctx.files())) + @classmethod def put_attempt(cls, id: str, *, description: str, file_path: str, filename: str) -> Self: """``put_at`` under a prefix of this attempt's own (``ArtifactStore.attempt_prefix``), so a write that diff --git a/src/agent_env/artifact/artifacts/file_artifact_universe.py b/src/agent_env/artifact/artifacts/file_artifact_universe.py index 600ecd91..8dddbd0d 100644 --- a/src/agent_env/artifact/artifacts/file_artifact_universe.py +++ b/src/agent_env/artifact/artifacts/file_artifact_universe.py @@ -9,7 +9,7 @@ from pydantic import ConfigDict, Field, model_serializer -from agent_env.artifact.artifact import _write_twin +from agent_env.artifact.artifact import ArtifactFolder, _write_twin from agent_env.artifact.ref import ArtifactRef from agent_env.artifact.universe import Universe from agent_env.config import get_config @@ -176,6 +176,11 @@ def from_toml(cls, data: dict, ctx: AuthoringContext) -> "FileArtifactUniverse": ctx.accept(data, **cls.toml_keys) return cls.put_bundled(ctx.id, files=ctx.files()) + @classmethod + def bundle_folder(cls, ctx: AuthoringContext) -> ArtifactFolder: + """What ``from_toml`` reads from the folder: every file it holds.""" + return ArtifactFolder(ctx.files()) + @classmethod def put_existing( cls, diff --git a/src/agent_env/artifact/artifacts/skill.py b/src/agent_env/artifact/artifacts/skill.py index 175d7b4d..6cabf63f 100644 --- a/src/agent_env/artifact/artifacts/skill.py +++ b/src/agent_env/artifact/artifacts/skill.py @@ -17,7 +17,7 @@ import yaml from pydantic import ConfigDict, Field, model_serializer -from agent_env.artifact.artifact import Artifact, _write_twin +from agent_env.artifact.artifact import Artifact, ArtifactFolder, _write_twin from agent_env.config import get_config from agent_env.store.base import ObjectNotFoundError from agent_env.store.ids import derive_id @@ -132,6 +132,11 @@ def from_toml(cls, data: Any, ctx: AuthoringContext) -> "SkillArtifact": """Write the skill in the bundle folder: every file its folder walk lists.""" return cls.put(ctx.id, files=ctx.files()) + @classmethod + def bundle_folder(cls, ctx: AuthoringContext) -> ArtifactFolder: + """What ``from_toml`` reads from the folder: every file it holds.""" + return ArtifactFolder(ctx.files()) + def get_skill_files(self) -> "FileArtifactUniverse": from agent_env.artifact.artifacts.file_artifact_universe import FileArtifactUniverse return FileArtifactUniverse.get(self.skill_files_id) diff --git a/src/agent_env/bundle/authoring.py b/src/agent_env/bundle/authoring.py index dba27711..7b0c46fd 100644 --- a/src/agent_env/bundle/authoring.py +++ b/src/agent_env/bundle/authoring.py @@ -11,24 +11,22 @@ from typing import Any, NoReturn, TypeVar, overload from agent_env.artifact.artifact import Artifact -from agent_env.artifact.artifacts.environment import EnvironmentArtifact -from agent_env.artifact.artifacts.environment_universe import EnvironmentUniverseArtifact from agent_env.entity_refs import EntityKind, parse_toml_ref from agent_env.env.env import Env from agent_env.env.envs._deployment import provider_refusal from agent_env.providers.env_providers.constants import GATEWAY_SERVICE_NAMES from agent_env.providers.env_providers.env_gateway_provider import EnvironmentGatewayProvider from agent_env.providers.env_providers.env_provider import _env_provider_class -from agent_env.store.ids import validate_local_id from agent_env.utils.card_naming import card_names_in_files -from ._fs import fold, os_reason, relative, show, with_article +from ._fs import os_reason, relative, show, with_article from .parse import CONFIG_FILES, LEAVINGS, NAMED_BY, Bundle, BundleEntry, BundleError A = TypeVar("A", bound=Artifact) E = TypeVar("E", bound=Env) _BRING_IT_IN = "copy what it points to into the bundle, or put it in a store and refer to it by id" +_AS_UNIVERSE = "leave out its declared type to write the folder as a file_artifact_universe" _TOML_TYPES = {str: "a string", dict: "a table", list: "a list"} @@ -59,8 +57,8 @@ def name(self) -> str: def dir(self) -> Path: return self.entry.path - def files(self) -> dict[str, Path]: - return entry_files(self.bundle, self.entry) + def files(self, *, empty_ok: bool = False) -> dict[str, Path]: + return entry_files(self.bundle, self.entry, empty_ok=empty_ok) def try_files(self) -> dict[str, Path] | None: """``files``, or None when the folder can't be listed, which the plan's walk reports.""" @@ -72,8 +70,20 @@ def try_files(self) -> dict[str, Path] | None: def file(self) -> tuple[str, Path]: return entry_file(self.bundle, self.entry) - def universe(self) -> UniverseLayout: - return universe_layout(self.bundle, self.entry) + def only(self, files: dict[str, Path], remedy: str = _AS_UNIVERSE) -> dict[str, Path]: + """``files``, which must be one file. Raises BundleError otherwise, suggesting ``remedy``.""" + return _only(self.bundle, self.entry, files, remedy) + + def listing(self) -> tuple[dict[str, Path], set[tuple[str, ...]]]: + """The folder's ``files``, none at all included, and every folder under it, by every path to it, as the names + on that path in NFC.""" + walk = _Walk(self.bundle, self.entry, empty_ok=True) + return walk.files(), walk.folders + + @property + def leavings(self) -> tuple[str, ...]: + """The names a folder walk leaves out, which the OS or Python leaves behind.""" + return tuple(sorted(LEAVINGS)) def renamed(self, data: dict, names: Mapping[str, str]) -> tuple[dict, list[str]]: """``data`` less each key ``names`` maps to the key an author writes instead, and a problem for each: @@ -184,6 +194,12 @@ def refuse(self, problems: list[str]) -> NoReturn: where = relative(self.bundle.root, self.entry.path) raise BundleError([f"{where}: {problem}" for problem in problems]) + def refuse_within(self, problems: list[str]) -> NoReturn: + """Raise BundleError with ``problems``, each ``: ``, named by its path in the + bundle.""" + where = relative(self.bundle.root, self.entry.path) + raise BundleError([f"{where}/{problem}" for problem in problems]) + @overload def env(self, ref: str | dict[str, Any]) -> Env: ... @overload @@ -244,9 +260,6 @@ def entry_file(bundle: Bundle, entry: BundleEntry) -> tuple[str, Path]: return key.rsplit("/", 1)[-1], path -_AS_UNIVERSE = "leave out its declared type to write the folder as a file_artifact_universe" - - def one_file(bundle: Bundle, entry: BundleEntry, remedy: str = _AS_UNIVERSE) -> dict[str, Path]: """``entry_files`` of a folder that must hold one file. Raises BundleError otherwise, suggesting ``remedy``.""" return _only(bundle, entry, entry_files(bundle, entry), remedy) @@ -260,145 +273,9 @@ def _only(bundle: Bundle, entry: BundleEntry, files: dict[str, Path], remedy: st return files -# The keys that name the file an environment artifact wraps: ``file``, and the stored document's names for it, which -# its toml check refuses with a pointer to ``file``. -_NAMES_ITS_FILE = ("file", *(stored for stored, key in EnvironmentArtifact.toml_stored_names.items() if key == "file")) - - -def environment_files(bundle: Bundle, entry: BundleEntry) -> dict[str, Path]: - """What an environment artifact's write reads from its folder: its one file, or none when its artifact.toml's - ``file`` names the artifact to wrap instead. Raises BundleError otherwise.""" - files = entry_files(bundle, entry, empty_ok=True) - if not (isinstance(entry.config, dict) and any(name in entry.config for name in _NAMES_ITS_FILE)): - if not files: - raise BundleError([f"{relative(bundle.root, entry.path)}: has no file for the environment to wrap " - f"({', '.join(sorted(LEAVINGS))} don't count); put its one file here, or name the " - "artifact to wrap with file"]) - return _only(bundle, entry, files, "name the artifact to wrap with file, or give each other file an artifact " - "folder of its own") - if files: - listed = ", ".join(repr(name) for name in files) - raise BundleError([f"{relative(bundle.root, entry.path)}: its file is the artifact file names, so the folder " - f"holds only artifact.toml, and it also has {listed}; give the file an artifact folder of " - "its own, or leave out file to wrap the folder's one file"]) - return {} - - -# The folder of a universe's metadata files, as `environment-universe get --output-dir` writes them. -UNIVERSE_METADATA = EnvironmentUniverseArtifact.metadata_name - - -@dataclass(frozen=True) -class LaidOut: - """One file of a universe folder, and the ids it's written under.""" - - name: str # its environment's name, or its metadata key - filename: str - path: Path - file_id: str - environment_id: str | None # None for a metadata file - - -@dataclass(frozen=True) -class UniverseLayout: - """An environment_universe folder, laid out as ``environment-universe get --output-dir`` writes one: each - ``/`` folder holds one environment's file, and each ``metadata//`` folder one metadata - file. ``files`` lists them all, as ``entry_files`` does.""" - - environments: tuple[LaidOut, ...] - metadata: tuple[LaidOut, ...] - files: dict[str, Path] - - -def universe_layout(bundle: Bundle, entry: BundleEntry) -> UniverseLayout: - """The environments and metadata files an environment_universe's folder holds, each with the ids it's - written under: ``__`` over ``____file`` for an environment, and - ``__metadata__`` for a metadata file. Raises BundleError listing every problem.""" - files = entry_files(bundle, entry, empty_ok=True) - where = relative(bundle.root, entry.path) - top = _folders(entry.path) - # A metadata/ folder spelled in another case is a typo, as for every reserved name in a bundle. - near = sorted(name for name in top if name != UNIVERSE_METADATA and fold(name) == fold(UNIVERSE_METADATA)) - problems = [f"{where}/{name}: rename to {UNIVERSE_METADATA}; names are case-sensitive, and an environment named " - f"{name} goes in an artifact folder of its own, named in environment_artifacts" for name in near] - # Every folder counts, so one left without its file is refused rather than dropped. - folders: dict[tuple[bool, str], list[str]] = {(False, name): [] for name in top - if name != UNIVERSE_METADATA and name not in near} - folders.update({(True, name): [] for name in _folders(entry.path / UNIVERSE_METADATA)}) - for key in files: - parts = key.split("/") - if parts[0] in near: - continue - metadata = parts[0] == UNIVERSE_METADATA - if len(parts) == (3 if metadata else 2): - folders.setdefault((metadata, parts[-2]), []).append(key) - elif len(parts) == 1: - problems.append(f"{where}/{key}: a universe folder holds only artifact.toml and a folder for each " - "environment; move it into /, or give it an artifact folder of its own") - elif metadata and len(parts) == 2: - problems.append(f"{where}/{key}: {UNIVERSE_METADATA}/ holds a folder for each key, with that key's one " - f"file in it ({UNIVERSE_METADATA}//); an environment named " - f"{UNIVERSE_METADATA} goes in an artifact folder of its own, named in " - "environment_artifacts") - else: - folder = "/".join(parts[:2]) if metadata else parts[0] - what = "a metadata key's" if metadata else "an environment's" - problems.append(f"{where}/{key}: {what} folder holds its one file directly, {folder}/, with no " - "folders inside") - environments, metadata_files = [], [] - for (metadata, name), keys in sorted(folders.items()): - folder = f"{where}/{UNIVERSE_METADATA}/{name}" if metadata else f"{where}/{name}" - if len(keys) != 1: - listed = f" ({', '.join(repr(key.rsplit('/', 1)[-1]) for key in keys)})" if keys else "" - problems.append(f"{folder}: holds one file, and has {len(keys)}{listed}") - continue - # Renaming an environment's folder renames the environment, so say how to keep its name. - rename = "rename the folder" if metadata else ( - f"rename the folder, which renames the environment, or keep the name by giving the environment an " - f'artifact folder of its own, with environment_name = "{name}", named in environment_artifacts') - if name != name.strip(): - problems.append(f"{folder}: a name that opens or closes with a space can't name the artifacts written " - "from it; rename the folder") - continue - if "__" in name: - problems.append(f"{folder}: a name holding __ could clash with the ids derived from it; {rename}") - continue - if metadata: - environment_id, file_id = None, EnvironmentUniverseArtifact.derived_metadata_id(entry.id, name) - else: - environment_id = EnvironmentUniverseArtifact.derived_environment_id(entry.id, name) - file_id = EnvironmentArtifact.derived_file_id(environment_id) - try: - for id in (environment_id, file_id): - if id is not None: - validate_local_id(id) - except ValueError as e: - problems.append(f"{folder}: can't name the artifacts written from it ({e}); {rename}") - continue - laid = LaidOut(name, keys[0].rsplit("/", 1)[-1], files[keys[0]], file_id, environment_id) - (metadata_files if metadata else environments).append(laid) - if problems: - raise BundleError(problems) - return UniverseLayout(tuple(environments), tuple(metadata_files), files) - - -def _folders(path: Path) -> list[str]: - """The names of the folders directly in ``path``, as the walk spells them, or none when it isn't one.""" - try: - with os.scandir(path) as listing: - return [unicodedata.normalize("NFC", child.name) for child in listing - if child.name not in LEAVINGS and child.is_dir()] - except OSError: - return [] # the walk reports why it can't be listed - - -def universe_files(bundle: Bundle, entry: BundleEntry) -> dict[str, Path]: - """What an environment_universe's write reads from its folder: every file ``universe_layout`` lays out.""" - return universe_layout(bundle, entry).files - - class _Walk: - """Lists an entry's folder, following a link only when its target is inside the bundle.""" + """Lists an entry's folder, following a link only when its target is inside the bundle. ``folders`` then holds + every folder under it, by every path to it, as the names on that path in NFC.""" def __init__(self, bundle: Bundle, entry: BundleEntry, empty_ok: bool = False): self.bundle = bundle @@ -407,6 +284,7 @@ def __init__(self, bundle: Bundle, entry: BundleEntry, empty_ok: bool = False): self.root = _identity(bundle.root) self.problems: list[str] = [] self.found: dict[str, Path] = {} + self.folders: set[tuple[str, ...]] = set() def files(self) -> dict[str, Path]: top = self.entry.path @@ -452,6 +330,7 @@ def _walk(self, top: Path) -> None: if identity in holders: self._problem(path, "links back to a folder that holds it") else: + self.folders.add(tuple(unicodedata.normalize("NFC", part) for part in (*parts, child.name))) pending.append((path, (*parts, child.name), holders | {identity})) elif kind == stat.S_IFREG: if os.access(path, os.R_OK): diff --git a/src/agent_env/bundle/ledger.py b/src/agent_env/bundle/ledger.py index 6f98b1cc..03768fd9 100644 --- a/src/agent_env/bundle/ledger.py +++ b/src/agent_env/bundle/ledger.py @@ -33,7 +33,7 @@ from .authoring import build_context_files from .parse import Bundle, BundleKind -from .plan import Plan, Write, env_writer, folder_walk, unpinned_store_refs +from .plan import Plan, Write, artifact_writer, env_writer, folder_files, unpinned_store_refs from .resolve import BuiltImage, BundleImage, ReferencedImage LEDGER_COLLECTION = "bundle_ledger" @@ -127,7 +127,7 @@ def digest(self, write: Write, needs: Mapping[tuple[str, str], int]) -> Digest | "files": {}, "needs": {}, "store_refs": {}} if write.kind in (BundleKind.ARTIFACT, BundleKind.SKILL) and not isinstance(write.source, ReferencedImage): built = isinstance(write.source, BuiltImage) - listing = build_context_files if built else folder_walk(get_artifact_registry().get(_type(write))) + listing = build_context_files if built else folder_files for key, path in listing(self._plan.bundle.bundle, write.source.entry).items(): inputs["files"][key] = _file_sha256(path) + (_mode(path) if built else "") if built and write.id in self._contexts: @@ -207,16 +207,17 @@ def materializing(bundle: Bundle, on_wait: Callable[[], None] | None = None) -> def _tracked(write: Write) -> bool: - """Whether the ledger can list everything ``write`` is made from. Not yet for a type with a ``from_toml`` of - its own, which may read its folder in ways it can't see. An artifact, a skill included, is made from its toml, - the files its type reads from its folder (``folder_walk``) and what it references; an agent or env from its + """Whether the ledger can list everything ``write`` is made from. Not for an artifact type whose ``from_toml`` + doesn't come with a ``bundle_folder`` saying what it reads, nor yet for an env type with a ``from_toml`` of its + own, which may read its folder in ways it can't see. An artifact, a skill included, is made from its toml, + the files its type reads from its folder (``bundle_folder``) and what it references; an agent or env from its toml and what it references. A built image is made from its build context, ``build_context_files``; what the build fetches (its base image, packages) isn't an input. A referenced image is made from its ref's text, so a tag that moves in its registry isn't a change. An image folder's image is also made from its description.""" if isinstance(write.source, BundleImage): return True if write.kind in (BundleKind.ARTIFACT, BundleKind.SKILL): - return folder_walk(get_artifact_registry().get(_type(write))) is not None + return artifact_writer(get_artifact_registry().get(_type(write))) if write.kind is BundleKind.ENV: return env_writer(get_env_registry().get(_type(write))) return True diff --git a/src/agent_env/bundle/materialize.py b/src/agent_env/bundle/materialize.py index aca81887..af025ae3 100644 --- a/src/agent_env/bundle/materialize.py +++ b/src/agent_env/bundle/materialize.py @@ -42,7 +42,7 @@ from .authoring import AuthoringContext, build_context_files from .ledger import Ledger, materializing from .parse import BundleError, BundleKind -from .plan import Plan, Write, env_writer, folder_walk, unpinned_store_refs +from .plan import Plan, Write, artifact_writer, env_writer, unpinned_store_refs from .resolve import BuiltImage, BundleImage, ReferencedImage, build_step @dataclass(frozen=True) @@ -310,7 +310,7 @@ def _unwritable(write: Write) -> str | None: if write.kind not in _WRITERS: return with_article(write.kind.value.removesuffix("s")) type_ = write.source.entry.type - if write.kind is BundleKind.ARTIFACT and folder_walk(get_artifact_registry().get(canonical_type(type_))) is None: + if write.kind is BundleKind.ARTIFACT and not artifact_writer(get_artifact_registry().get(canonical_type(type_))): return with_article(f"{type_} artifact") if write.kind is BundleKind.ENV and not env_writer(get_env_registry().get(type_)): return with_article(f"{type_} env") diff --git a/src/agent_env/bundle/plan.py b/src/agent_env/bundle/plan.py index 961bf28e..72684af0 100644 --- a/src/agent_env/bundle/plan.py +++ b/src/agent_env/bundle/plan.py @@ -11,20 +11,14 @@ import heapq import unicodedata -from collections.abc import Sequence +from collections.abc import Mapping, Sequence from dataclasses import dataclass +from pathlib import Path from typing import Any from agent_env.a2a_agent import A2AAgent -from agent_env.artifact import ( - Artifact, - CliArtifact, - EnvironmentArtifact, - EnvironmentUniverseArtifact, - FileArtifact, - FileArtifactUniverse, - SkillArtifact, -) +from agent_env.artifact import Artifact, CliArtifact, EnvironmentArtifact +from agent_env.artifact.artifact import ArtifactFolder from agent_env.artifact.registry import canonical_type, get_artifact_registry from agent_env.config import get_config from agent_env.entity_refs import EntityKind @@ -51,16 +45,8 @@ from agent_env.task_step.task_steps.load_artifact import LoadArtifactTaskStep from ._fs import fold, relative, with_article -from .authoring import ( - UNIVERSE_METADATA, - AuthoringContext, - entry_files, - environment_files, - one_file, - universe_files, - universe_layout, -) -from .parse import BundleEntry, BundleError, BundleKind +from .authoring import AuthoringContext +from .parse import Bundle, BundleEntry, BundleError, BundleKind from .resolve import BuiltImage, BundleImage, Reference, ResolvedBundle, ResolvedEntry, build_step _Key = tuple[str, str] # (store, id) @@ -68,9 +54,8 @@ STORE_GETTERS = {EntityKind.ENV: Env.get, EntityKind.AGENT: A2AAgent.get, EntityKind.ARTIFACT: Artifact.get, EntityKind.TASK: Task.get} -# The artifact types written from their folders, and what each one's write reads there, which the ledger hashes. -_FOLDER_WALKS = {FileArtifact: one_file, FileArtifactUniverse: entry_files, EnvironmentArtifact: environment_files, - EnvironmentUniverseArtifact: universe_files, SkillArtifact: entry_files, CliArtifact: entry_files} +# The type a field naming an environment artifact takes: what an artifact referencing one holds. +_ENVIRONMENT = EnvironmentArtifact.model_fields["type"].default # The env types whose from_toml reads only the toml and what it names, so a type built by one of them is written from # its env.toml and the ledger can list what it's made from. _ENV_FROM_TOMLS = (Env, MCPServerEnv, WebsiteEnv, MultiEnv) @@ -134,6 +119,7 @@ def __init__(self, resolved: ResolvedBundle): self.outputs = {(output.kind.value, output.id): (entry.entry, output) for entry in resolved.entries for output in entry.outputs} self.accepted: dict[_Key, dict] = {} # each toml a type checks, as it accepted it + self.folders: dict[_Key, ArtifactFolder | BundleError | None] = {} # each artifact's, as its type read it self.steps: dict[_Key, list[TaskStep]] = {} # each task's steps that build, as a write will build them def _add(self, key: _Key, source: ResolvedEntry | BundleImage, needs: list[_Key]) -> None: @@ -151,7 +137,7 @@ def plan(self, tasks: Sequence[str], evals: Sequence[str]) -> Plan: self._check_cli_loads(closure) self._check_env_types(closure) self._check_multi_names(closure, store_latest) - self._check_universes(closure, store_latest) + self._check_environment_names(closure, store_latest) self._check_loaded_environments(closure, store_latest) if self.problems: raise BundleError(self.problems) @@ -168,7 +154,7 @@ def check(self) -> None: self._build_tasks(everything) self._check_files(everything) self._check_cli_loads(everything) - self._check_universes(everything, None) + self._check_environment_names(everything, None) self._check_loaded_environments(everything, None) if self.problems: raise BundleError(self.problems) @@ -198,25 +184,34 @@ def _check_writers(self) -> None: writers[key] = writer def _written_beside(self, source: ResolvedEntry) -> list[tuple[str, str]]: - """The ids an environment or universe artifact writes besides its own, each with the folder it's written - from: the file an environment wraps from its folder, and each of a universe's environments and metadata - files. A folder that can't be written names none; the file check reports why.""" - if source.entry.kind is not BundleKind.ARTIFACT: - return [] - walk = folder_walk(get_artifact_registry().get(canonical_type(source.entry.type))) - bundle, where = self.resolved.bundle, self._path(source.entry) - try: - if walk is environment_files: - return ([(EnvironmentArtifact.derived_file_id(source.entry.id), where)] - if environment_files(bundle, source.entry) else []) - if walk is universe_files: - layout = universe_layout(bundle, source.entry) - return [*((id, f"{where}/{laid.name}") for laid in layout.environments - for id in (laid.environment_id, laid.file_id)), - *((laid.file_id, f"{where}/{UNIVERSE_METADATA}/{laid.name}") for laid in layout.metadata)] - except BundleError: - pass - return [] + """The ids an artifact writes besides its own, its folder's children, each with the folder it's written + from. A folder that can't be written names none; the file check reports why.""" + folder, where = self._folder(source.entry), self._path(source.entry) + return [(child.id, self._within(where, child.path)) for child in folder.children] if folder else [] + + def _folder(self, entry: BundleEntry) -> ArtifactFolder | None: + """What the artifact ``entry``'s type reads from its folder and writes besides it (``bundle_folder``), or + None for one its type doesn't write from a folder, or whose folder the file check refuses.""" + folder = self._read_folder(entry) + return folder if isinstance(folder, ArtifactFolder) else None + + def _read_folder(self, entry: BundleEntry) -> ArtifactFolder | BundleError | None: + """``_folder``, or the BundleError its type raised for the folder, read once.""" + if entry.kind not in (BundleKind.ARTIFACT, BundleKind.SKILL): + return None + key = _key(entry) + if key not in self.folders: + cls = get_artifact_registry().get(canonical_type(entry.type)) + try: + self.folders[key] = (cls.bundle_folder(AuthoringContext(self.resolved.bundle, entry)) + if artifact_writer(cls) else None) + except BundleError as e: + self.folders[key] = e + return self.folders[key] + + @staticmethod + def _within(where: str, path: str) -> str: + return f"{where}/{path}" if path else where def _check_loops(self) -> None: for component in _components(self.nodes, self.edges, self.rank): @@ -358,13 +353,10 @@ def _check_files(self, closure: set[_Key]) -> None: if source.entry.kind not in (BundleKind.ARTIFACT, BundleKind.SKILL): continue cls = registry.get(canonical_type(source.entry.type)) - walk = folder_walk(cls) - if walk is None: + if not artifact_writer(cls): continue - try: - walk(self.resolved.bundle, source.entry) - except BundleError as e: - self.problems.extend(e.problems) + if isinstance(folder := self._read_folder(source.entry), BundleError): + self.problems.extend(folder.problems) ctx = AuthoringContext(self.resolved.bundle, source.entry) accept = getattr(cls, "accept_toml", None) try: @@ -470,36 +462,34 @@ def _check_multi_names(self, closure: set[_Key], store_latest: dict[tuple[Entity if reason is not None: self.problems.append(f"{where}: env_provider_type {provider_type!r} {reason}; rename one") - def _check_universes(self, closure: set[_Key], store_latest: dict[tuple[EntityKind, str], int] | None) -> None: - """A universe needs an environment, and each of its environments a name of its own: one its folder holds is - named by its folder, one it names by its toml or, from the store, at the version the plan read. Without + def _check_environment_names(self, closure: set[_Key], + store_latest: dict[tuple[EntityKind, str], int] | None) -> None: + """The environments one artifact holds need names of their own: those its folder writes, and the environment + artifacts it references, named by their tomls or, from the store, at the version the plan read. Without ``store_latest`` (a check that reads no store), the store's environments aren't counted.""" - registry = get_artifact_registry() for key in sorted(closure, key=self.rank.__getitem__): source = self.nodes[key] - if isinstance(source, BundleImage) or source.entry.kind is not BundleKind.ARTIFACT: - continue - cls = registry.get(canonical_type(source.entry.type)) - if cls is None or not issubclass(cls, EnvironmentUniverseArtifact) or folder_walk(cls) is None: + if isinstance(source, BundleImage) or (folder := self._folder(source.entry)) is None: continue - try: - layout = universe_layout(self.resolved.bundle, source.entry) - except BundleError: - continue # reported by the file check where = self._path(source.entry) - taken = {laid.name: f"the {laid.name}/ folder" for laid in layout.environments} # who has each name - for ref in source.references: - name = self._environment_name(ref, store_latest) - if name is None: - continue + held = [*((self._within(where, child.path), child.environment_name) + for child in folder.children if child.environment_name), + *((f"{where}: {ref.where}", name) for ref, name in self._referenced(source, store_latest))] + taken: dict[str, str] = {} # who has each name + for who, name in held: if name in taken: - self.problems.append(f"{where}: {ref.where}: environment_name {name!r} is also {taken[name]}'s; " - "a universe's environments need names of their own") + self.problems.append(f"{who}: environment_name {name!r} is also {taken[name]}'s; " + f"{with_article(source.entry.type)}'s environments need names of their own") else: - taken[name] = ref.where - if not layout.environments and not source.references and not source.config.get("environment_artifacts"): - self.problems.append(f"{where}: a universe needs an environment: a folder of its own, named after it " - "and holding its file, or one environment_artifacts names") + taken[name] = who + + def _referenced(self, source: ResolvedEntry, + store_latest: dict[tuple[EntityKind, str], int] | None) -> list[tuple[Reference, str]]: + """Each environment artifact ``source`` references, with its environment_name when that's known.""" + return [(ref, name) for ref in source.references + if ref.kind is EntityKind.ARTIFACT and ref.artifact_type is not None + and canonical_type(ref.artifact_type) == _ENVIRONMENT + and (name := self._environment_name(ref, store_latest)) is not None] def _check_loaded_environments(self, closure: set[_Key], store_latest: dict[tuple[EntityKind, str], int] | None) -> None: @@ -560,27 +550,19 @@ def _server_names(self, ref: Reference, store_latest: dict[tuple[EntityKind, str return {name} if name else None def _seeded(self, ref: Reference, store_latest: dict[tuple[EntityKind, str], int] | None) -> list[tuple[str, str]]: - """Each environment the bundle's own environment or universe ``ref`` names seeds, as (where it's named, - its environment_name): a universe's folders and what its environment_artifacts names, or an environment's - own name. Nothing for a store artifact or another type.""" + """Each environment the bundle's own artifact ``ref`` names seeds, as (where it's named, its + environment_name): its own, the ones its folder writes and the environment artifacts it references. Nothing + for a store artifact, or one whose folder the file check refuses.""" entry = ref.local - if not isinstance(entry, BundleEntry) or entry.kind is not BundleKind.ARTIFACT: + if not isinstance(entry, BundleEntry) or (folder := self._folder(entry)) is None: return [] - cls = get_artifact_registry().get(canonical_type(entry.type)) - walk, where = folder_walk(cls), self._path(entry) - if walk is environment_files: - name = self.accepted.get(_key(entry), {}).get("environment_name") - return [(where, name)] if name else [] - if walk is not universe_files: - return [] - try: - layout = universe_layout(self.resolved.bundle, entry) - except BundleError: - return [] # reported by the file check - named = [(f"{where}: {child.where}", self._environment_name(child, store_latest)) - for child in self.nodes[_key(entry)].references] - return [*((f"{where}/{laid.name}", laid.name) for laid in layout.environments), - *((what, name) for what, name in named if name is not None)] + where, accepted = self._path(entry), self.accepted.get(_key(entry)) + own = accepted.get("environment_name") if isinstance(accepted, dict) else None + return [*([(where, own)] if own else []), + *((self._within(where, child.path), child.environment_name) + for child in folder.children if child.environment_name), + *((f"{where}: {named.where}", name) for named, name in self._referenced(self.nodes[_key(entry)], + store_latest))] def _environment_name(self, ref: Reference, store_latest: dict[tuple[EntityKind, str], int] | None) -> str | None: """The environment_name of the env or environment artifact ``ref`` names: a bundle one's from its accepted @@ -660,18 +642,21 @@ def env_writer(cls: type | None) -> bool: return cls is not None and next(c for c in cls.__mro__ if "from_toml" in vars(c)) in _ENV_FROM_TOMLS -def folder_walk(cls: type | None) -> Any: - """The listing a type's write runs: a subclass keeping its base's ``from_toml`` lists the same way, - one that overrides it may read its folder differently, so it gets none.""" - for base, walk in _FOLDER_WALKS.items(): - if cls is not None and issubclass(cls, base): - return walk if keeps_base_from_toml(cls, base) else None - return None +def artifact_writer(cls: type | None) -> bool: + """Whether an artifact of type ``cls`` is written from its bundle folder: by a ``from_toml`` whose class defines + ``bundle_folder`` too, so what the plan checks and the ledger hashes is what the write reads.""" + if cls is None: + return False + writer = next(c for c in cls.__mro__ if "from_toml" in vars(c)) + return writer is not Artifact and writer is next(c for c in cls.__mro__ if "bundle_folder" in vars(c)) -def keeps_base_from_toml(cls: type, base: type) -> bool: - """Whether ``cls`` is built from its TOML by ``base``'s ``from_toml`` rather than one of its own.""" - return next(c for c in cls.__mro__ if "from_toml" in vars(c)) is base +def folder_files(bundle: Bundle, entry: BundleEntry) -> Mapping[str, Path]: + """The files the write of the artifact ``entry`` reads from its folder, as its type's ``bundle_folder`` lists + them. Raises BundleError.""" + cls = get_artifact_registry()[canonical_type(entry.type)] + folder = cls.bundle_folder(AuthoringContext(bundle, entry)) + return folder.files if folder is not None else {} def _key(entity: BundleEntry | BundleImage) -> _Key: diff --git a/src/agent_env/cli/artifact/environment_universe.py b/src/agent_env/cli/artifact/environment_universe.py index ec6be818..5af6c718 100644 --- a/src/agent_env/cli/artifact/environment_universe.py +++ b/src/agent_env/cli/artifact/environment_universe.py @@ -4,6 +4,7 @@ import click from agent_env.artifact import FileArtifact, EnvironmentArtifact, EnvironmentUniverseArtifact +from agent_env.artifact.artifacts.environment_universe_folder import environment_path, metadata_path from agent_env.cli.utils import parse_artifact_ref @@ -126,9 +127,8 @@ def environment_universe_get( for sa in environment_artifacts: file_artifact = sa.get_file_artifact() data = file_artifact.load() - sa_dir = out / sa.environment_name - sa_dir.mkdir(parents=True, exist_ok=True) - dest = sa_dir / file_artifact.filename + dest = environment_path(out, sa.environment_name, file_artifact.filename) + dest.parent.mkdir(parents=True, exist_ok=True) dest.write_bytes(data) click.echo(f" {sa.environment_name} -> {dest}") @@ -137,9 +137,8 @@ def environment_universe_get( click.echo(f"Downloading {len(metadata_artifacts)} metadata file(s)...") for key, file_artifact in metadata_artifacts.items(): data = file_artifact.load() - metadata_dir = out / EnvironmentUniverseArtifact.metadata_name / key - metadata_dir.mkdir(parents=True, exist_ok=True) - dest = metadata_dir / file_artifact.filename + dest = metadata_path(out, key, file_artifact.filename) + dest.parent.mkdir(parents=True, exist_ok=True) dest.write_bytes(data) click.echo(f" {key} -> {dest}") diff --git a/tst/unit/artifact/environment_universe_folder_test.py b/tst/unit/artifact/environment_universe_folder_test.py new file mode 100644 index 00000000..d74f3e4f --- /dev/null +++ b/tst/unit/artifact/environment_universe_folder_test.py @@ -0,0 +1,55 @@ +"""The universe folder format on its own: ``read_folder`` reads back what ``environment_path`` and ``metadata_path`` +lay out, and names its problems by their paths in the folder.""" + +import hashlib +from pathlib import Path, PurePosixPath + +import pytest + +from agent_env.artifact.artifacts.environment_universe_folder import ( + UniverseFolderError, + environment_path, + metadata_path, + read_folder, +) + +UNIVERSE = "@local/~/b/u" + + +def _listing(paths): + """A folder's files keyed by their paths in it, and its folders, as a walk lists them.""" + files = {path.as_posix(): Path("/nowhere") / path for path in paths} + folders = {PurePosixPath(key).parents[i].parts for key in files for i in range(len(PurePosixPath(key).parts) - 1)} + return files, folders + + +def test_reading_a_folder_gives_back_every_name_and_key_its_paths_were_written_for(): + environments = {"gmail": "a.json", "drive/shared": "b.json", "g__mail": "c.json"} + keys = {"manifest": "m.json", "scenarios": "s.yaml", "scenarios/x.yaml": "x.yaml", "q&a?": "q.json"} + files, folders = _listing([environment_path(PurePosixPath(), name, filename) + for name, filename in environments.items()] + + [metadata_path(PurePosixPath(), key, filename) for key, filename in keys.items()]) + + read = read_folder(UNIVERSE, files, folders) + + assert {entry.name: entry.filename for entry in read.environments} == environments + assert {entry.name: entry.filename for entry in read.metadata} == keys + digest = hashlib.sha256(b"g__mail").hexdigest()[:16] + assert {entry.name: entry.environment_id for entry in read.environments}["g__mail"] == f"{UNIVERSE}__sha256__{digest}" + assert {entry.name: entry.file_id for entry in read.metadata}["scenarios/x.yaml"] == ( + f"{UNIVERSE}__metadata__scenarios/x.yaml") + + +def test_its_problems_name_paths_in_the_folder(): + files, folders = _listing([PurePosixPath("stray.json"), PurePosixPath("gmail/a.json"), + PurePosixPath("gmail/b.json")]) + + with pytest.raises(UniverseFolderError) as caught: + read_folder(UNIVERSE, files, folders | {("slack",)}) + + assert caught.value.problems == [ + "stray.json: a universe folder holds only artifact.toml and a folder for each environment; move it into " + "/, or give it an artifact folder of its own", + "gmail: holds one file, and has 2 ('a.json', 'b.json')", + "slack: holds one file, and has 0", + ] diff --git a/tst/unit/bundle/artifact_toml_test.py b/tst/unit/bundle/artifact_toml_test.py index 0dfa92f1..054d1ee3 100644 --- a/tst/unit/bundle/artifact_toml_test.py +++ b/tst/unit/bundle/artifact_toml_test.py @@ -4,6 +4,7 @@ what it records changes, and equal, field for field, to one the CLI writes from the same files. Each environment the bundle writes must seed a server of the env a task loads it into.""" +import hashlib import json import pytest @@ -20,6 +21,7 @@ from agent_env.cli import cli from agent_env.env.envs.mcp_server import MCPServerEnv from agent_env.env.envs.multi_env import MultiEnv +from agent_env.store import ids as ids_module from agent_env.store.routing import namespace_routing from tst.unit.bundle._support import RefusingStore, layout, local_store, plan_of @@ -260,6 +262,127 @@ def test_the_cli_names_the_files_it_writes_with_an_environment_and_a_universe_as assert EnvironmentUniverseArtifact.get("world").metadata_refs["manifest"].id == "world__metadata__manifest" +# Names and keys the CLI writes as nested folders, beside ones that hold __ or characters no @local id can. +_NAMES = ("gmail", "drive/shared", "g__mail", "crm: v2", "slack ") +_KEYS = ("manifest", "scenarios", "scenarios/a.yaml", "scenarios/b.yaml", "deep/x/y/z.json", "m__k", "q&a?") + + +def test_a_universe_the_cli_downloads_reads_back_whatever_its_names_and_keys_hold(bundle_dir, tmp_path): + for i, name in enumerate(_NAMES): + (tmp_path / f"env{i}.json").write_text(json.dumps({"environment": name})) + put = CliRunner().invoke(cli, ["artifact", "environment", "put", str(tmp_path / f"env{i}.json"), "--id", + f"env{i}", "--description", "d", "--environment-name", name]) + assert put.exit_code == 0, put.output + for i, key in enumerate(_KEYS): + (tmp_path / f"meta{i}.json").write_text(json.dumps({"key": key})) + put = CliRunner().invoke(cli, [ + "artifact", "environment-universe", "put", "--id", "original", + *(arg for i in range(len(_NAMES)) for arg in ("--environment-artifact", f"env{i}")), + *(arg for i, key in enumerate(_KEYS) for arg in ("--metadata", f"{key}={tmp_path / f'meta{i}.json'}"))]) + assert put.exit_code == 0, put.output + got = CliRunner().invoke(cli, ["artifact", "environment-universe", "get", "--id", "original", "--output-dir", + str(bundle_dir / "artifacts/copy")]) + assert got.exit_code == 0, got.output + _bundle(bundle_dir, _universe("copy"), "copy") + + first = _summary(_run(bundle_dir)) + + with namespace_routing(): + original, copy = EnvironmentUniverseArtifact.get("original"), EnvironmentUniverseArtifact.get(f"{ROOT}/copy") + assert _canonical(copy) == _canonical(original) + assert set(copy.get_metadata()) == set(_KEYS) + assert first[f"{ROOT}/copy"] == (1, False, ("new",)) + assert all(reused for _, reused, _ in _summary(_run(bundle_dir)).values()) + (bundle_dir / "artifacts/copy/metadata/deep/x/y/z.json/meta4.json").write_text('{"key": 2}') + assert _summary(_run(bundle_dir))[f"{ROOT}/copy"] == ( + 2, False, ("files changed: metadata/deep/x/y/z.json/meta4.json",)) + + +def test_a_name_an_id_can_hold_keeps_its_id_and_any_other_is_written_under_its_hash(bundle_dir): + _bundle(bundle_dir, _universe("w") | { + "artifacts/w/gmail/a.json": "{}", "artifacts/w/drive/shared/b.json": "{}", "artifacts/w/g__mail/c.json": "{}", + "artifacts/w/metadata/scenarios/x.yaml/x.yaml": "x: 1\n", "artifacts/w/metadata/q&a?/q.json": "{}", + }, "w") + + _run(bundle_dir) + + def hashed(name): + return f"sha256__{hashlib.sha256(name.encode()).hexdigest()[:16]}" + + world = EnvironmentUniverseArtifact.get(f"{ROOT}/w") + assert sorted((env.environment_name, env.id, env.get_file_artifact().id) + for env in world.get_environment_artifacts()) == [ + ("drive/shared", f"{ROOT}/w__drive/shared", f"{ROOT}/w__drive/shared__file"), + ("g__mail", f"{ROOT}/w__{hashed('g__mail')}", f"{ROOT}/w__{hashed('g__mail')}__file"), + ("gmail", f"{ROOT}/w__gmail", f"{ROOT}/w__gmail__file"), + ] + assert {key: file.id for key, file in world.get_metadata().items()} == { + "scenarios/x.yaml": f"{ROOT}/w__metadata__scenarios/x.yaml", + "q&a?": f"{ROOT}/w__metadata__{hashed('q&a?')}", + } + + +def test_a_name_spelled_like_another_names_hash_keeps_an_id_of_its_own(bundle_dir): + digest = hashlib.sha256("q&a?".encode()).hexdigest()[:16] + _bundle(bundle_dir, _universe("w") | { + "artifacts/w/gmail/a.json": "{}", f"artifacts/w/{digest}/b.json": "{}", "artifacts/w/q&a?/c.json": "{}", + f"artifacts/w/metadata/{digest}/m.json": "{}", "artifacts/w/metadata/q&a?/n.json": "{}", + }, "w") + + _run(bundle_dir) + + world = EnvironmentUniverseArtifact.get(f"{ROOT}/w") + assert {env.environment_name: env.id for env in world.get_environment_artifacts()} == { + "gmail": f"{ROOT}/w__gmail", digest: f"{ROOT}/w__{digest}", "q&a?": f"{ROOT}/w__sha256__{digest}"} + assert {key: file.id for key, file in world.get_metadata().items()} == { + digest: f"{ROOT}/w__metadata__{digest}", "q&a?": f"{ROOT}/w__metadata__sha256__{digest}"} + + +def test_a_universe_whose_id_leaves_no_room_for_a_hashed_childs_id_is_refused_naming_the_fix(bundle_dir, + monkeypatch): + # Room for gmail's ids beside the universe's, and none for a hashed name's, which are longer. + monkeypatch.setattr(ids_module, "MAX_LOCAL_ID_BYTES", len(f"{ROOT}/x__gmail__file")) + layout(bundle_dir, _refused(_universe("x") | {"artifacts/x/gmail/a.json": "{}", "artifacts/x/q&a?/b.json": "{}"})) + + (problem,) = _problems(bundle_dir) + assert problem.startswith("artifacts/x/q&a?: can't name the artifacts written from it (local id is ") + assert problem.endswith("); give the universe a shorter name") + assert not local_store().path.exists() + + +def test_metadata_in_another_case_is_refused_even_as_a_link_to_an_environments_folder(bundle_dir): + layout(bundle_dir, _refused(_universe("x") | {"artifacts/x/gmail/a.json": "{}"})) + (bundle_dir / "artifacts/x/Metadata").symlink_to("gmail") + + assert _problems(bundle_dir) == ( + "artifacts/x/Metadata: rename to metadata; names are case-sensitive, and an environment named Metadata goes " + "in an artifact folder of its own, named in environment_artifacts",) + + +def test_a_folder_holding_its_own_file_and_nested_ones_is_an_entry_of_each(bundle_dir): + _bundle(bundle_dir, _universe("w") | { + "artifacts/w/crm/a.json": "{}", "artifacts/w/crm/eu/b.json": "{}", "artifacts/w/metadata/k/k.json": "{}", + "artifacts/w/metadata/k/sub/s.json": "{}", "artifacts/w/metadata/only/folders/here/h.json": "{}", + }, "w") + + _run(bundle_dir) + + world = EnvironmentUniverseArtifact.get(f"{ROOT}/w") + assert sorted(env.environment_name for env in world.get_environment_artifacts()) == ["crm", "crm/eu"] + assert {key: file.filename for key, file in world.get_metadata().items()} == { + "k": "k.json", "k/sub": "s.json", "only/folders/here": "h.json"} + + +def test_a_nested_folder_holding_two_files_is_refused_naming_it(bundle_dir): + layout(bundle_dir, _refused(_universe("x") | {"artifacts/x/gmail/a.json": "{}", + "artifacts/x/metadata/scenarios/x.yaml/one.yaml": "", + "artifacts/x/metadata/scenarios/x.yaml/two.yaml": ""})) + + assert _problems(bundle_dir) == ( + "artifacts/x/metadata/scenarios/x.yaml: holds one file, and has 2 ('one.yaml', 'two.yaml')",) + assert not local_store().path.exists() + + def test_an_environment_written_either_way_equals_the_one_the_cli_writes(bundle_dir, tmp_path): (tmp_path / "data.json").write_text('{"rows": 1}\n') put = CliRunner().invoke(cli, ["artifact", "environment", "put", str(tmp_path / "data.json"), "--id", "original", @@ -321,21 +444,6 @@ def _refused(files): "into /, or give it an artifact folder of its own"), (_universe("x") | {"artifacts/x/gmail/a.json": "{}", "artifacts/x/gmail/b.json": "{}"}, "artifacts/x/gmail: holds one file, and has 2 ('a.json', 'b.json')"), - (_universe("x") | {"artifacts/x/gmail/inner/a.json": "{}"}, - "artifacts/x/gmail/inner/a.json: an environment's folder holds its one file directly, gmail/, with no " - "folders inside"), - (_universe("x") | {"artifacts/x/g__mail/a.json": "{}"}, - "artifacts/x/g__mail: a name holding __ could clash with the ids derived from it; rename the folder, which " - "renames the environment, or keep the name by giving the environment an artifact folder of its own, with " - 'environment_name = "g__mail", named in environment_artifacts'), - (_universe("x") | {"artifacts/x/gmail/a.json": "{}", "artifacts/x/metadata/m__k/m.json": "{}"}, - "artifacts/x/metadata/m__k: a name holding __ could clash with the ids derived from it; rename the folder"), - (_universe("x") | {"artifacts/x/slack /a.json": "{}"}, - "artifacts/x/slack : a name that opens or closes with a space can't name the artifacts written from it; rename " - "the folder"), - (_universe("x") | {"artifacts/x/ slack/a.json": "{}"}, - "artifacts/x/ slack: a name that opens or closes with a space can't name the artifacts written from it; rename " - "the folder"), (_universe("x") | {"artifacts/x/gmail/a.json": "{}", "artifacts/x/metadata/m.json": "{}"}, "artifacts/x/metadata/m.json: metadata/ holds a folder for each key, with that key's one file in it " "(metadata//); an environment named metadata goes in an artifact folder of its own, named in " @@ -343,23 +451,18 @@ def _refused(files): (_universe("x") | {"artifacts/x/gmail/a.json": "{}", "artifacts/x/Metadata/k/m.json": "{}"}, "artifacts/x/Metadata: rename to metadata; names are case-sensitive, and an environment named Metadata goes in " "an artifact folder of its own, named in environment_artifacts"), - (_universe("x") | {"artifacts/x/gmail/a.json": "{}", "artifacts/x/metadata/k/sub/m.json": "{}"}, - "artifacts/x/metadata/k/sub/m.json: a metadata key's folder holds its one file directly, metadata/k/, " - "with no folders inside"), (_universe("x", environment_artifacts='["gmail-data"]') | {"artifacts/x/gmail/a.json": "{}"} | _environment("gmail-data", "gmail") | {"artifacts/gmail-data/a.json": "{}"}, - "artifacts/x: environment_artifacts[0]: environment_name 'gmail' is also the gmail/ folder's; a universe's " - "environments need names of their own"), + "artifacts/x: environment_artifacts[0]: environment_name 'gmail' is also artifacts/x/gmail's; an " + "environment_universe's environments need names of their own"), (_universe("x", service_artifact_refs='["a"]', metadata='{ m = "f" }') | {"artifacts/x/gmail/a.json": "{}"}, "artifacts/x: artifact.toml: service_artifact_refs is what the stored document calls it; write " "environment_artifacts instead"), ], ids=["environment-two-files", "environment-no-file", "environment-no-name", "environment-empty-name", "environment-service-name", "environment-file-artifact-id", "environment-file-and-description", "environment-file-and-a-file", "environment-over-a-universe", "universe-empty", "universe-root-file", - "universe-two-files", "universe-nested-folder", "universe-double-underscore", - "universe-metadata-double-underscore", "universe-trailing-space", "universe-leading-space", - "universe-flat-metadata", "universe-metadata-in-another-case", "universe-nested-metadata", - "universe-repeated-name", "universe-stored-names"]) + "universe-two-files", "universe-flat-metadata", "universe-metadata-in-another-case", "universe-repeated-name", + "universe-stored-names"]) def test_an_artifact_toml_that_cant_be_written_is_refused_before_any_write(bundle_dir, files, problem): layout(bundle_dir, _refused(files)) @@ -382,7 +485,7 @@ def test_a_universe_folder_left_without_its_file_is_refused_rather_than_dropped( assert sorted(_problems(bundle_dir)) == [ "artifacts/x/drive: holds one file, and has 0", - "artifacts/x/gmail-old: holds one file, and has 0", + "artifacts/x/gmail-old/inner: holds one file, and has 0", "artifacts/x/metadata/manifest: holds one file, and has 0", "artifacts/x/slack: holds one file, and has 0", ] @@ -490,8 +593,9 @@ def test_a_universe_naming_a_store_environment_with_the_name_of_one_of_its_folde _bundle(bundle_dir, {**_universe("world", environment_artifacts='["mail-data"]'), "artifacts/world/gmail/a.json": "{}"}, "world") - assert _problems(bundle_dir) == ("artifacts/world: environment_artifacts[0]: environment_name 'gmail' is also the " - "gmail/ folder's; a universe's environments need names of their own",) + assert _problems(bundle_dir) == ("artifacts/world: environment_artifacts[0]: environment_name 'gmail' is also " + "artifacts/world/gmail's; an environment_universe's environments need names of " + "their own",) def test_a_check_reports_artifact_toml_problems_without_reading_a_store(bundle_dir, monkeypatch): diff --git a/tst/unit/bundle/bundle_folder_test.py b/tst/unit/bundle/bundle_folder_test.py new file mode 100644 index 00000000..e749d877 --- /dev/null +++ b/tst/unit/bundle/bundle_folder_test.py @@ -0,0 +1,151 @@ +"""A bundle writes an artifact type from its folder through the type's own ``bundle_folder`` and ``from_toml``, so a +plugin's type is planned, checked, written and reused as a core type is.""" + +import json +from typing import Literal + +import pytest +from pydantic import Field + +from agent_env.artifact import Artifact, FileArtifact +from agent_env.artifact.artifact import ArtifactFolder, FolderChild +from agent_env.artifact.ref import ArtifactRef +from agent_env.bundle import BundleError +from agent_env.bundle.materialize import materialize +from agent_env.config.runtime import Config +from tst.unit.bundle._support import layout, plan_of + +ROOT = "@local/~/triage" +TOML = {"artifacts/rooms/artifact.toml": 'type = "rooms_folder_test"\n'} + + +class _Rooms(Artifact): + """A plugin's type, no core type's subclass: each file seeds the environment it's named after, written as a file + artifact of its own beside the rooms.""" + + type: Literal["rooms_folder_test"] = "rooms_folder_test" + rooms: dict[str, ArtifactRef] = Field(default_factory=dict) + + @classmethod + def bundle_folder(cls, ctx): + files = ctx.files() + return ArtifactFolder(files, tuple(FolderChild(f"{ctx.id}-{key}", key, environment_name=key.split(".")[0]) + for key in files)) + + @classmethod + def from_toml(cls, data, ctx): + folder = cls.bundle_folder(ctx) + rooms = {child.environment_name: FileArtifact.put_attempt( + child.id, description=child.path, file_path=str(folder.files[child.path]), filename=child.path).as_ref() + for child in folder.children} + return cls.put(id=ctx.id, rooms=rooms) + + +class _OwnFile(FileArtifact): + """A plugin's file type with a from_toml and a bundle_folder of its own, keeping only the first line.""" + + type: Literal["own_file_folder_test"] = "own_file_folder_test" + + @classmethod + def bundle_folder(cls, ctx): + return ArtifactFolder(ctx.only(ctx.files())) + + @classmethod + def from_toml(cls, data, ctx): + [(name, path)] = cls.bundle_folder(ctx).files.items() + return cls.put_attempt(ctx.id, description=path.read_text().splitlines()[0], file_path=str(path), + filename=name) + + +@pytest.fixture(autouse=True) +def registries(monkeypatch): + artifacts = {**Config().artifact_registry(), **{cls.model_fields["type"].default: cls for cls in (_Rooms, _OwnFile)}} + monkeypatch.setattr(Config, "artifact_registry", lambda self: artifacts) + + +@pytest.fixture +def bundle_dir(tmp_path, monkeypatch, local_stores, cli_routing): + monkeypatch.setenv("HOME", str(tmp_path)) + return tmp_path / "triage" + + +def _sandbox_loading(files, *artifacts): + steps = [{"id": "box", "type": "deploy_sandbox", "sandbox_name": "box", "sandbox_mode": "vm", + "sandbox_type": "local"}] + steps += [{"id": f"load-{name}", "type": "load_artifact", "sandbox_name": "box", "artifact_id": name, + "destination_path": f"/app/{name}", "depends_on": ["box"]} for name in artifacts] + return {**files, "tasks/t.json": json.dumps(steps)} + + +def _env_loading(files, artifact, env): + return {**files, f"envs/{env}/Dockerfile": "FROM scratch\n", f"envs/{env}/env.toml": f'environment_name = "{env}"\n', + "tasks/t.json": json.dumps([ + {"id": "env", "type": "deploy_env", "env_id": env}, + {"id": "load", "type": "load_artifact", "env_id": env, "artifact_id": artifact, "depends_on": ["env"]}, + ])} + + +def _summary(materialization): + return {done.write.id: (done.version, done.reused) for done in materialization.writes} + + +def _problems(root): + with pytest.raises(BundleError) as caught: + plan_of(root) + return caught.value.problems + + +def test_a_plugin_type_is_written_with_its_children_and_rewritten_only_when_a_file_it_reads_changes(bundle_dir): + layout(bundle_dir, _sandbox_loading({**TOML, "artifacts/rooms/crm.json": "{}", "artifacts/rooms/slack.json": "[]"}, + "rooms")) + + assert _summary(materialize(plan_of(bundle_dir)))[f"{ROOT}/rooms"] == (1, False) + rooms = Artifact.get(f"{ROOT}/rooms") + assert {name: ref.id for name, ref in rooms.rooms.items()} == {"crm": f"{ROOT}/rooms-crm.json", + "slack": f"{ROOT}/rooms-slack.json"} + assert FileArtifact.get(f"{ROOT}/rooms-slack.json").filename == "slack.json" + assert _summary(materialize(plan_of(bundle_dir)))[f"{ROOT}/rooms"] == (1, True) + (bundle_dir / "artifacts/rooms/slack.json").write_text("[1]") + assert _summary(materialize(plan_of(bundle_dir)))[f"{ROOT}/rooms"] == (2, False) + + +def test_a_plugin_types_folder_problem_is_reported_before_anything_is_written(bundle_dir): + layout(bundle_dir, _sandbox_loading(TOML, "rooms")) + (bundle_dir / "artifacts/rooms/sub").mkdir() + + assert _problems(bundle_dir) == ("artifacts/rooms: has no files to write (.DS_Store, Thumbs.db, __pycache__, " + "desktop.ini don't count)",) + + +def test_a_child_whose_id_another_write_claims_is_refused(bundle_dir): + layout(bundle_dir, _sandbox_loading({**TOML, "artifacts/rooms/crm.json": "{}", + "artifacts/rooms-crm.json/a.json": "{}"}, "rooms", "rooms-crm.json")) + + assert _problems(bundle_dir) == (f"artifacts/rooms-crm.json: '{ROOT}/rooms-crm.json' is also written by the " + "artifact written from artifacts/rooms/crm.json; give each its own id",) + + +def test_a_child_seeding_no_server_of_the_env_it_loads_into_is_refused(bundle_dir): + layout(bundle_dir, _env_loading({**TOML, "artifacts/rooms/crn.json": "{}"}, "rooms", "crm")) + + assert _problems(bundle_dir) == ( + "tasks/t.json: step 'load': artifacts/rooms/crn.json seeds an environment named 'crn', and envs/crm has no " + "server by that name ('crm'); name it after the server it seeds, or leave it out",) + + +def test_two_children_seeding_one_environment_are_refused(bundle_dir): + layout(bundle_dir, _sandbox_loading({**TOML, "artifacts/rooms/crm.json": "{}", "artifacts/rooms/crm.yaml": "{}"}, + "rooms")) + + assert _problems(bundle_dir) == ("artifacts/rooms/crm.yaml: environment_name 'crm' is also " + "artifacts/rooms/crm.json's; a rooms_folder_test's environments need names of " + "their own",) + + +def test_a_core_types_subclass_with_both_hooks_of_its_own_is_written_by_them(bundle_dir): + layout(bundle_dir, _sandbox_loading({"artifacts/note/artifact.toml": 'type = "own_file_folder_test"\n', + "artifacts/note/n.txt": "first\nsecond\n"}, "note")) + + assert _summary(materialize(plan_of(bundle_dir)))[f"{ROOT}/note"] == (1, False) + assert Artifact.get(f"{ROOT}/note").description == "first" + assert _summary(materialize(plan_of(bundle_dir)))[f"{ROOT}/note"] == (1, True)