PineForge HPO 1.11.0
Native hyperparameter optimization for PineForge strategies
Loading...
Searching...
No Matches
artifact.py
Go to the documentation of this file.
1"""Content-addressed PineScript strategy artifact builder."""
2
3from __future__ import annotations
4
5from contextlib import AbstractContextManager
6import copy
7import ctypes
8from dataclasses import dataclass
9from datetime import datetime, timezone
10import hashlib
11import json
12import os
13from pathlib import Path
14import platform
15import re
16import shutil
17import subprocess
18import sys
19import tempfile
20import time
21from typing import Any, Callable, Literal, Mapping, Sequence
22
23from .error import HpoError
24
25from .transpile import (
26 REQUEST_FEED_INVENTORY_SCHEMA,
27 CodegenIdentity,
28 TranspileDiagnostic,
29 TranspileFailure,
30 codegen_identity,
31 codegen_supports_request_inventory,
32 legacy_input_kind_version,
33 transpile_source,
34 wrapper_codegen_refusal,
35)
36from .window_plan import (
37 WindowPlanError,
38 WindowPlanIdentity,
39 _associate_window_plan_path,
40 _check_compile_identity,
41 _require_engine_layout_identity,
42 associate_window_plan,
43 derive_window_plan_path,
44 require_window_plan_block,
45 resolve_window_plan_path,
46 verify_window_plan,
47)
48
49
50
51
52CANONICAL_COMPILE_FLAGS = (
53 "-std=c++17",
54 "-O2",
55 "-ffp-contract=off",
56 "-fPIC",
57 "-shared",
58)
59# Engine 1.0 requires Clang builds outside CMake to lift Clang's 256-level bracket
60# limit: generated C++ for long Pine chains nests deeper. GCC has no such flag.
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81class ArtifactBuildError(HpoError, RuntimeError):
82 """A configuration, transpile, compile, inventory, or publication failure."""
83
85 self,
86 stage: str,
87 message: str,
88 *,
89 command: Sequence[str] = (),
90 stdout: str = "",
91 stderr: str = "",
92 diagnostics: Sequence[TranspileDiagnostic] = (),
93 actual_sha256: str | None = None,
94 expected_sha256: str | None = None,
95 ) -> None:
96 self.stage = stage
97 self.command = tuple(command)
98 self.stdout = stdout
99 self.stderr = stderr
100 self.diagnostics = tuple(diagnostics)
101 self.actual_sha256 = actual_sha256
102 self.expected_sha256 = expected_sha256
103 super().__init__(message)
104 code, args = {
105 "configuration": ("hpo_toolchain_unavailable", {"reason": "configuration"}),
106 "transpile": ("hpo_transpile_failed", {}),
107 "compile": ("hpo_toolchain_unavailable", {"reason": "compile"}),
108 "inventory": (
109 "hpo_toolchain_unavailable",
110 {"reason": "transpile_internal"},
111 ),
112 "plugin_validation": ("hpo_plugin_invalid", {"reason": "validation"}),
113 "cache": ("hpo_output_io_failed", {}),
114 "generated_cpp_hash": ("hpo_generated_cpp_hash_mismatch", {}),
115 }.get(stage, ("hpo_unclassified_error", {}))
116 self.with_failure(code, args)
117 if stage == "transpile":
118 if any(item.phase == "CONFIGURATION" for item in self.diagnostics):
119 self.with_failure(
120 "hpo_toolchain_unavailable", {"reason": "configuration"}
121 )
122 elif any(item.phase == "INTERNAL" for item in self.diagnostics):
123 self.with_failure(
124 "hpo_toolchain_unavailable", {"reason": "transpile_internal"}
125 )
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167@dataclass(frozen=True)
169 """Paths and metadata consumed by the native strategy-plugin loader.
170
171 A deferred build can return a pending artifact. Read ``plugin_validation``
172 in the recorded manifest and provenance: ``{"status": "deferred"}`` means
173 pending until ``ArtifactBuilder.validate_built_plugin`` succeeds. Successful
174 validation records ``abi_version`` and ``required_symbols`` instead.
175 """
176
177 artifact_key: str
178 request_key: str
179 plugin_path: Path
180 generated_cpp_path: Path
181 manifest_path: Path
182 provenance_path: Path
183 cache_hit: bool
184 inputs: tuple[dict[str, Any], ...]
185 strategy_params: Mapping[str, Any]
186
187 @property
188 def library_path(self) -> Path:
189 """Compatibility alias for callers that call plugins libraries."""
190
191 return self.plugin_path
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
384 """Transpile and compile a Pine strategy once per compatible request identity.
385
386 Compilation has no time limit by default. Set ``compile_timeout_seconds`` to
387 a positive number to limit it; ``None`` means no limit. Artifact lock waits
388 default to 120 seconds; a lock that never frees is a deadlock to report.
389 ``plugin_load="deferred"`` skips loading the compiled plugin during a build;
390 call ``validate_built_plugin`` later in the process chosen by the caller.
391 ``wrapper_launched=True`` requires stable codegen >= 1.10.0 and refuses
392 source-build cache returns that skip transpilation or compilation.
393 """
394
396 self,
397 *,
398 engine_root: str | os.PathLike[str],
399 cache_dir: str | os.PathLike[str] | None = None,
400 compiler: str | os.PathLike[str] | None = None,
401 eigen_include: str | os.PathLike[str] | None = None,
402 plugin_validator: Callable[[Path, int], Mapping[str, Any]] | None = None,
403 plugin_load: Literal["build", "deferred"] = "build",
404 lock_timeout_seconds: float = 120.0,
405 compile_timeout_seconds: float | None = None,
406 wrapper_launched: bool = False,
407 ) -> None:
408 self.engine_root = Path(engine_root).expanduser().resolve()
409 self.cache_dir = (
410 Path(cache_dir).expanduser().resolve()
411 if cache_dir
412 else _default_cache_dir()
413 )
414 self.compiler = os.fspath(compiler) if compiler is not None else None
416 Path(eigen_include).expanduser().resolve()
417 if eigen_include is not None
418 else None
419 )
420 self.plugin_validator = plugin_validator or self._validate_plugin
421 self.plugin_load = plugin_load
422 self.lock_timeout_seconds = lock_timeout_seconds
423 self.compile_timeout_seconds = compile_timeout_seconds
424 if type(wrapper_launched) is not bool:
425 raise ArtifactBuildError("configuration", "wrapper_launched must be a bool")
426 self.wrapper_launched = wrapper_launched
427
428
429
430
431
432
433
434
435
436
437
438
439 def build(self, pine_source: str, *, filename: str = "<input>") -> StrategyArtifact:
440 if not isinstance(pine_source, str):
441 raise TypeError("pine_source must be a string")
442 self._check_build_configuration()
443 checked_codegen = (
444 _require_wrapper_codegen_identity() if self.wrapper_launched else None
445 )
446 engine = self._resolve_engine_layout()
447 compiler = self._resolve_compiler_identity()
448 eigen = self._resolve_eigen_identity(engine)
449 try:
450 codegen = checked_codegen or codegen_identity()
451 except ModuleNotFoundError as error:
452 raise ArtifactBuildError(
453 "configuration",
454 "pineforge-codegen is required to compile PineScript; install "
455 "pineforge-hpo[transpile] or provide a precompiled artifact",
456 ) from error
457 output_extension, link_mode = self._platform_link_mode()
458 request_identity = self._request_identity(
459 source_sha256=hashlib.sha256(pine_source.encode("utf-8")).hexdigest(),
460 codegen=codegen,
461 engine=engine,
462 compiler=compiler,
463 eigen=eigen,
464 output_extension=output_extension,
465 link_mode=link_mode,
466 inventory_schema=(
467 REQUEST_FEED_INVENTORY_SCHEMA
468 if codegen_supports_request_inventory()
469 else None
470 ),
471 )
472 request_key = _json_sha256(request_identity)
473
474 hit = self._lookup_request(
475 request_key,
476 request_identity,
477 output_extension,
478 allow_deferred=self.plugin_load == "deferred",
479 )
480 if hit is not None:
481 if self.wrapper_launched:
482 raise _wrapper_bypass_error("request_cache")
483 return hit
484
485 lock_path = self.cache_dir / "locks" / f"{request_key}.lock"
486 with _KeyLock(lock_path, self.lock_timeout_seconds):
487 hit = self._lookup_request(
488 request_key, request_identity, output_extension, allow_deferred=True
489 )
490 if hit is not None:
491 if self.wrapper_launched:
492 raise _wrapper_bypass_error("request_cache_locked")
493 if self.plugin_load == "build":
494 return self._validate_built_plugin_locked(hit)
495 return hit
496
497 transpile_result = transpile_source(pine_source, filename=filename)
498 try:
499 generated_cpp = transpile_result.require_success()
500 except TranspileFailure as error:
501 raise ArtifactBuildError(
502 "transpile",
503 str(error),
504 diagnostics=transpile_result.diagnostics,
505 ) from error
506
507 if transpile_result.input_kind_schema == 1 and legacy_input_kind_version(
508 request_identity["codegen"]["version"]
509 ):
510 raise ArtifactBuildError(
511 "transpile",
512 "input_kind_schema: 1 contradicts recorded codegen version; "
513 "codegen >= 1.1.0 is required",
514 )
515
516 generated_cpp_sha256 = hashlib.sha256(
517 generated_cpp.encode("utf-8")
518 ).hexdigest()
519 artifact_identity = {
520 "request_identity": request_identity,
521 "generated_cpp_sha256": generated_cpp_sha256,
522 }
523 artifact_key = _json_sha256(artifact_identity)
524 existing = self._load_artifact(
525 artifact_key=artifact_key,
526 request_key=request_key,
527 request_identity=request_identity,
528 output_extension=output_extension,
529 cache_hit=True,
530 allow_deferred=True,
531 )
532 if existing is not None:
533 if self.wrapper_launched:
534 raise _wrapper_bypass_error("artifact_cache")
535 if self.plugin_load == "build":
536 existing = self._validate_built_plugin_locked(existing)
537 self._write_request_index(request_key, artifact_key)
538 return existing
539
540 artifact = self._compile_and_publish(
541 artifact_key=artifact_key,
542 request_key=request_key,
543 request_identity=request_identity,
544 output_extension=output_extension,
545 generated_cpp=generated_cpp,
546 generated_cpp_sha256=generated_cpp_sha256,
547 inputs=transpile_result.inputs,
548 input_kind_schema=transpile_result.input_kind_schema,
549 strategy_params=transpile_result.strategy_params,
550 diagnostics=transpile_result.diagnostics,
551 request_feed_inventory=transpile_result.request_feed_inventory,
552 source_name=filename,
553 compiler=compiler,
554 engine=engine,
555 eigen=eigen,
556 )
557 self._write_request_index(request_key, artifact_key)
558 return artifact
559
561 self,
562 generated_cpp: bytes,
563 *,
564 expected_sha256: str,
565 inputs: Sequence[Mapping[str, Any]],
566 input_kind_schema: int | None,
567 strategy_params: Mapping[str, Any],
568 request_feed_inventory: Mapping[str, Any] | None,
569 diagnostics: Sequence[TranspileDiagnostic] = (),
570 filename: str = "<input>",
571 ) -> StrategyArtifact:
572 """Compile checked UTF-8 C++ bytes with the caller's emission metadata.
573
574 Verify the raw bytes before resolving a compiler or publishing anything.
575 This entry never transpiles and never returns before a real compile.
576 Supply metadata from the checked emission; no inventory is inferred.
577 """
578
579 if not isinstance(generated_cpp, bytes):
580 raise _checked_cpp_error("bytes", "generated_cpp must be raw bytes")
581 if (
582 not isinstance(expected_sha256, str)
583 or re.fullmatch(r"[0-9a-fA-F]{64}", expected_sha256) is None
584 ):
585 raise _checked_cpp_error(
586 "expected_sha256", "expected_sha256 must contain exactly 64 hex digits"
587 )
588 actual_sha256 = hashlib.sha256(generated_cpp).hexdigest()
589 if actual_sha256 != expected_sha256.lower():
590 raise ArtifactBuildError(
591 "generated_cpp_hash",
592 "checked generated C++ SHA-256 mismatch: "
593 f"expected {expected_sha256}, actual {actual_sha256}",
594 actual_sha256=actual_sha256,
595 expected_sha256=expected_sha256,
596 )
597 try:
598 generated_text = generated_cpp.decode("utf-8")
599 except UnicodeDecodeError as error:
600 raise _checked_cpp_error(
601 "utf8", f"checked generated C++ is not strict UTF-8: {error}"
602 ) from error
603
604 self._check_build_configuration()
605 try:
606 codegen = (
607 _require_wrapper_codegen_identity()
608 if self.wrapper_launched
609 else codegen_identity()
610 )
611 except ModuleNotFoundError as error:
612 raise ArtifactBuildError(
613 "configuration",
614 "checked C++ requires its emitting codegen installation",
615 ) from error
616 if (
617 not isinstance(inputs, (list, tuple))
618 or any(not isinstance(item, Mapping) for item in inputs)
619 or not isinstance(strategy_params, Mapping)
620 or not (
621 input_kind_schema is None
622 or (type(input_kind_schema) is int and input_kind_schema == 1)
623 )
624 or not isinstance(filename, str)
625 or (
626 request_feed_inventory is not None
627 and (
628 not isinstance(request_feed_inventory, Mapping)
629 or request_feed_inventory.get("schema")
630 != REQUEST_FEED_INVENTORY_SCHEMA
631 or request_feed_inventory.get("source_sha256") != actual_sha256
632 )
633 )
634 or (request_feed_inventory is None and codegen_supports_request_inventory())
635 ):
636 raise _checked_cpp_error(
637 "metadata",
638 "checked emission requires its real inputs, input-kind schema, "
639 "strategy parameters and source-matching request-feed inventory "
640 "(when the loaded producer supports one)",
641 )
642 if not isinstance(diagnostics, (list, tuple)) or any(
643 not isinstance(item, TranspileDiagnostic) for item in diagnostics
644 ):
645 raise _checked_cpp_error(
646 "diagnostics", "checked diagnostics must be TranspileDiagnostic objects"
647 )
648 if any(item.severity == "error" for item in diagnostics):
649 raise ArtifactBuildError(
650 "transpile",
651 "checked emission has error diagnostics",
652 diagnostics=diagnostics,
653 )
654 if input_kind_schema == 1 and legacy_input_kind_version(codegen.version):
655 raise _checked_cpp_error(
656 "metadata", "input_kind_schema: 1 requires codegen >= 1.1.0"
657 )
658 # Freeze caller-owned metadata before deriving its identity or compiling.
659 try:
660 emission = copy.deepcopy(
661 {
662 "inputs": [dict(item) for item in inputs],
663 "input_kind_schema": input_kind_schema,
664 "strategy_params": dict(strategy_params),
665 "request_feed_inventory": (
666 dict(request_feed_inventory)
667 if request_feed_inventory is not None
668 else None
669 ),
670 "diagnostics": [item.to_dict() for item in diagnostics],
671 "source_name": filename,
672 }
673 )
674 emission_sha256 = _json_sha256(emission)
675 except (TypeError, ValueError) as error:
676 raise _checked_cpp_error(
677 "metadata",
678 f"checked emission metadata is not JSON-serializable: {error}",
679 ) from error
680 engine = self._resolve_engine_layout()
681 compiler = self._resolve_compiler_identity()
682 eigen = self._resolve_eigen_identity(engine)
683 output_extension, link_mode = self._platform_link_mode()
684 request_identity = self._request_identity(
685 source_sha256=actual_sha256,
686 codegen=codegen,
687 engine=engine,
688 compiler=compiler,
689 eigen=eigen,
690 output_extension=output_extension,
691 link_mode=link_mode,
692 inventory_schema=(
693 REQUEST_FEED_INVENTORY_SCHEMA
694 if request_feed_inventory is not None
695 else None
696 ),
697 )
698 request_identity["source_kind"] = "checked_cpp"
699 request_identity["emission_metadata_sha256"] = emission_sha256
700 request_key = _json_sha256(request_identity)
701 artifact_key = _json_sha256(
702 {
703 "request_identity": request_identity,
704 "generated_cpp_sha256": actual_sha256,
705 }
706 )
707 lock_path = self.cache_dir / "locks" / f"{request_key}.lock"
708 with _KeyLock(lock_path, self.lock_timeout_seconds):
709 artifact = self._compile_and_publish(
710 artifact_key=artifact_key,
711 request_key=request_key,
712 request_identity=request_identity,
713 output_extension=output_extension,
714 generated_cpp=generated_text,
715 generated_cpp_sha256=actual_sha256,
716 inputs=emission["inputs"],
717 input_kind_schema=input_kind_schema,
718 strategy_params=emission["strategy_params"],
719 diagnostics=diagnostics,
720 request_feed_inventory=emission["request_feed_inventory"],
721 source_name=filename,
722 compiler=compiler,
723 engine=engine,
724 eigen=eigen,
725 )
726 self._write_request_index(request_key, artifact_key)
727 return artifact
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770 def validate_built_plugin(self, artifact: StrategyArtifact) -> StrategyArtifact:
771 """Load and validate a pending source-cache plugin in the calling process.
772
773 Call this in a fresh process to separate the first plugin load from
774 compilation. This method starts no process and needs neither codegen nor
775 a local engine/compiler installation. It checks the recorded engine ABI
776 and content hashes under the request lock before publishing success.
777 Already validated artifacts are checked without loading or rewriting.
778 The caller supplies process isolation; this method does not install a jail.
779 """
780
781 if self.lock_timeout_seconds <= 0:
782 raise ArtifactBuildError("configuration", "build timeouts must be positive")
783 keys = (artifact.request_key, artifact.artifact_key)
784 if any(
785 len(key) != 64
786 or any(character not in "0123456789abcdef" for character in key)
787 for key in keys
788 ):
789 raise ArtifactBuildError(
790 "cache", "validation requires valid source-cache keys"
791 )
792 artifact_dir = self.cache_dir / "artifacts" / artifact.artifact_key
793 output_extension, _ = self._platform_link_mode()
794 expected_paths = (
795 artifact_dir / "manifest.json",
796 artifact_dir / "provenance.json",
797 artifact_dir / "generated.cpp",
798 artifact_dir / f"strategy{output_extension}",
799 )
800 actual_paths = (
801 artifact.manifest_path,
802 artifact.provenance_path,
803 artifact.generated_cpp_path,
804 artifact.plugin_path,
805 )
806 if actual_paths != expected_paths or any(
807 path.resolve().parent != artifact_dir for path in actual_paths
808 ):
809 raise ArtifactBuildError(
810 "cache", "only this builder's source-cache artifacts may be validated"
811 )
812 lock_path = self.cache_dir / "locks" / f"{artifact.request_key}.lock"
813 with _KeyLock(lock_path, self.lock_timeout_seconds):
814 return self._validate_built_plugin_locked(artifact)
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
886 self,
887 artifact: StrategyArtifact,
888 *,
889 explicit_path: str | os.PathLike[str] | None = None,
890 ) -> dict[str, Any]:
891 """Admit a source-cache artifact for selected execution under its request lock.
892
893 Ordinary builds do not call this method. An existing block is re-resolved
894 and verified without changing its expected hash. An interrupted two-file
895 enrichment leaves selected admission refused until enrichment is re-run.
896 A retry repairs missing or stale metadata, including a stale request_identity,
897 from the validating side under the same lock. A validated old cache with a
898 matching local engine may acquire metadata, never new plugin bytes or keys.
899 An explicit path also supports the non-derivable third archive layout.
900 Arbitrary precompiled artifacts cannot acquire an identity here.
901 """
902
903 keys = (artifact.request_key, artifact.artifact_key)
904 if any(
905 len(key) != 64
906 or any(character not in "0123456789abcdef" for character in key)
907 for key in keys
908 ):
909 raise WindowPlanError(
910 "window_plan_identity",
911 "enrichment requires valid source-cache keys",
912 )
913 artifact_dir = self.cache_dir / "artifacts" / artifact.artifact_key
914 output_extension, _ = self._platform_link_mode()
915 expected_paths = (
916 artifact_dir / "manifest.json",
917 artifact_dir / "provenance.json",
918 artifact_dir / "generated.cpp",
919 artifact_dir / f"strategy{output_extension}",
920 )
921 actual_paths = (
922 artifact.manifest_path,
923 artifact.provenance_path,
924 artifact.generated_cpp_path,
925 artifact.plugin_path,
926 )
927 if actual_paths != expected_paths or any(
928 path.resolve().parent != artifact_dir for path in actual_paths
929 ):
930 raise WindowPlanError(
931 "window_plan_identity",
932 "only this builder's validated source-cache artifacts may be enriched",
933 )
934 lock_path = self.cache_dir / "locks" / f"{artifact.request_key}.lock"
935 with _KeyLock(lock_path, self.lock_timeout_seconds):
936 try:
937 manifest = json.loads(
938 artifact.manifest_path.read_text(encoding="utf-8"),
939 )
940 provenance = json.loads(
941 artifact.provenance_path.read_text(encoding="utf-8")
942 )
943 except (OSError, ValueError) as error:
944 raise WindowPlanError(
945 "window_plan_manifest_invalid",
946 f"cannot read cached manifest/provenance: {artifact_dir}: {error}",
947 ) from error
948 if not isinstance(manifest, dict) or not isinstance(provenance, dict):
949 raise WindowPlanError(
950 "window_plan_manifest_invalid",
951 "source-cache manifest and provenance must be JSON objects",
952 )
953 # Shallow copies suffice: repairs replace top-level values and never
954 # mutate shared nested objects in place.
955 originals = (dict(manifest), dict(provenance))
956 block = None
957 request_identity = manifest.get("request_identity")
958 if "window_plan" in manifest or "window_plan" in provenance:
959 candidates = []
960 # Repair only from a candidate whose block and file-local
961 # request_identity validate.
962 for location, document in (
963 ("manifest", manifest),
964 ("provenance", provenance),
965 ):
966 if "window_plan" not in document:
967 continue
968 try:
969 identity = WindowPlanIdentity.from_dict(document["window_plan"])
970 _check_compile_identity(
971 identity,
972 document.get("request_identity"),
973 location,
974 )
975 except WindowPlanError:
976 continue
977 candidates.append((identity, document))
978 if not candidates:
979 raise WindowPlanError(
980 "window_plan_manifest_invalid",
981 "no cached window_plan block validates against its "
982 "file's request_identity",
983 )
984 identity = candidates[0][0]
985 if any(candidate != identity for candidate, _ in candidates[1:]):
986 raise WindowPlanError(
987 "window_plan_identity",
988 "manifest.window_plan differs from provenance.window_plan",
989 )
990 block = identity.to_dict()
991 # A valid identity in a file without a valid block cannot rescue a
992 # block stored with a stale identity; ordinary builds treat that
993 # stale request identity as a cache miss.
994 request_identity = next(
995 (
996 document["request_identity"]
997 for _, document in candidates
998 if _json_sha256(document["request_identity"])
999 == artifact.request_key
1000 ),
1001 None,
1002 )
1003 if (
1004 not isinstance(request_identity, Mapping)
1005 or _json_sha256(request_identity) != artifact.request_key
1006 ):
1007 raise WindowPlanError(
1008 "window_plan_identity",
1009 "source-cache request_identity differs from its request_key",
1010 )
1011 if block is not None:
1012 for document in (manifest, provenance):
1013 document["window_plan"] = block
1014 if document.get("request_identity") != request_identity:
1015 document["request_identity"] = request_identity
1016 if document is provenance:
1017 document["compile"] = request_identity["compile"]
1018 # Validate the proposed repair before publishing either metadata file.
1019 cached = self._load_artifact(
1020 artifact_key=artifact.artifact_key,
1021 request_key=artifact.request_key,
1022 request_identity=request_identity,
1023 output_extension=output_extension,
1024 cache_hit=True,
1025 metadata=(manifest, provenance),
1026 allow_deferred=True,
1027 )
1028 if cached is None:
1029 raise ArtifactBuildError(
1030 "cache",
1031 f"cannot enrich an invalid source-cache artifact: {artifact_dir}",
1032 )
1033 if manifest["plugin_validation"] == {"status": "deferred"}:
1034 raise ArtifactBuildError(
1035 "plugin_validation",
1036 "artifact plugin validation is pending; "
1037 "call validate_built_plugin first",
1038 )
1039 try:
1040 engine = self._resolve_engine_layout()
1041 except (ArtifactBuildError, OSError, RuntimeError, ValueError) as error:
1042 raise WindowPlanError(
1043 "window_plan_identity",
1044 f"cannot verify local engine identity: {self.engine_root}: {error}",
1045 ) from error
1046 _require_engine_layout_identity(engine, request_identity)
1047 if block is not None:
1048 path = resolve_window_plan_path(explicit_path, self.engine_root, block)
1049 verify_window_plan(block, path)
1050 elif explicit_path is not None:
1051 block = _associate_window_plan_path(engine, explicit_path)
1052 else:
1053 block = associate_window_plan(engine)
1054 if block is None:
1055 if derive_window_plan_path(engine) is None:
1056 raise WindowPlanError(
1057 "window_plan_path",
1058 "the selected archive requires an explicit absolute "
1059 "window_plan path for source-cache enrichment",
1060 )
1061 raise WindowPlanError(
1062 "window_plan_missing",
1063 "the selected engine archive has no window_plan helper sibling",
1064 )
1065 manifest["window_plan"] = block
1066 provenance["window_plan"] = block
1067 require_window_plan_block(manifest, provenance)
1068 try:
1069 for destination, payload, original in zip(
1070 (artifact.manifest_path, artifact.provenance_path),
1071 (manifest, provenance),
1072 originals,
1073 ):
1074 # Equal metadata is not rewritten just to canonicalize key
1075 # order; readers access fields by key.
1076 if payload != original:
1077 self._write_window_plan_metadata(destination, payload)
1078 except OSError as error:
1079 raise ArtifactBuildError(
1080 "cache",
1081 f"cannot publish window_plan metadata: {artifact_dir}: {error}",
1082 ) from error
1083 return block
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724def build_strategy_artifact(
1725 pine_source: str,
1726 *,
1727 engine_root: str | os.PathLike[str],
1728 cache_dir: str | os.PathLike[str] | None = None,
1729 compiler: str | os.PathLike[str] | None = None,
1730 eigen_include: str | os.PathLike[str] | None = None,
1731 plugin_load: Literal["build", "deferred"] = "build",
1732 wrapper_launched: bool = False,
1733 filename: str = "<input>",
1734) -> StrategyArtifact:
1735 """Convenience entry point for a one-off CLI artifact build."""
1736
1737 builder = ArtifactBuilder(
1738 engine_root=engine_root,
1739 cache_dir=cache_dir,
1740 compiler=compiler,
1741 eigen_include=eigen_include,
1742 plugin_load=plugin_load,
1743 wrapper_launched=wrapper_launched,
1744 )
1745 return builder.build(pine_source, filename=filename)
1746
1747
1748
1749
1750
1751
1752
1753
1754
None __init__(self, str stage, str message, *Sequence[str] command=(), str stdout="", str stderr="", Sequence[TranspileDiagnostic] diagnostics=(), str|None actual_sha256=None, str|None expected_sha256=None)
Definition artifact.py:95
None __init__(self, *str|os.PathLike[str] engine_root, str|os.PathLike[str]|None cache_dir=None, str|os.PathLike[str]|None compiler=None, str|os.PathLike[str]|None eigen_include=None, Callable[[Path, int], Mapping[str, Any]]|None plugin_validator=None, Literal["build", "deferred"] plugin_load="build", float lock_timeout_seconds=120.0, float|None compile_timeout_seconds=None, bool wrapper_launched=False)
Definition artifact.py:407
StrategyArtifact build(self, str pine_source, *str filename="<input>")
Definition artifact.py:439
StrategyArtifact validate_built_plugin(self, StrategyArtifact artifact)
Definition artifact.py:770
dict[str, Any] enrich_window_plan(self, StrategyArtifact artifact, *str|os.PathLike[str]|None explicit_path=None)
Definition artifact.py:890
StrategyArtifact build_cpp(self, bytes generated_cpp, *str expected_sha256, Sequence[Mapping[str, Any]] inputs, int|None input_kind_schema, Mapping[str, Any] strategy_params, Mapping[str, Any]|None request_feed_inventory, Sequence[TranspileDiagnostic] diagnostics=(), str filename="<input>")
Definition artifact.py:571
with_failure(self, str|None code, dict[str, Any]|None args, str origin="hpo")
Definition error.py:25