398 engine_root: str | os.PathLike[str],
399 cache_dir: str | os.PathLike[str] |
None =
None,
400 compiler: str | os.PathLike[str] |
None =
None,
401 eigen_include: str | os.PathLike[str] |
None =
None,
402 plugin_validator: Callable[[Path, int], Mapping[str, Any]] |
None =
None,
403 plugin_load: Literal[
"build",
"deferred"] =
"build",
404 lock_timeout_seconds: float = 120.0,
405 compile_timeout_seconds: float |
None =
None,
406 wrapper_launched: bool =
False,
410 Path(cache_dir).expanduser().resolve()
412 else _default_cache_dir()
414 self.
compiler = os.fspath(compiler)
if compiler
is not None else None
416 Path(eigen_include).expanduser().resolve()
417 if eigen_include
is not None
424 if type(wrapper_launched)
is not bool:
439 def build(self, pine_source: str, *, filename: str =
"<input>") -> StrategyArtifact:
440 if not isinstance(pine_source, str):
441 raise TypeError(
"pine_source must be a string")
442 self._check_build_configuration()
446 engine = self._resolve_engine_layout()
447 compiler = self._resolve_compiler_identity()
448 eigen = self._resolve_eigen_identity(engine)
450 codegen = checked_codegen
or codegen_identity()
451 except ModuleNotFoundError
as error:
454 "pineforge-codegen is required to compile PineScript; install "
455 "pineforge-hpo[transpile] or provide a precompiled artifact",
457 output_extension, link_mode = self._platform_link_mode()
458 request_identity = self._request_identity(
459 source_sha256=hashlib.sha256(pine_source.encode(
"utf-8")).hexdigest(),
464 output_extension=output_extension,
467 REQUEST_FEED_INVENTORY_SCHEMA
468 if codegen_supports_request_inventory()
472 request_key = _json_sha256(request_identity)
474 hit = self._lookup_request(
482 raise _wrapper_bypass_error(
"request_cache")
485 lock_path = self.
cache_dir /
"locks" / f
"{request_key}.lock"
487 hit = self._lookup_request(
488 request_key, request_identity, output_extension, allow_deferred=
True
492 raise _wrapper_bypass_error(
"request_cache_locked")
494 return self._validate_built_plugin_locked(hit)
497 transpile_result = transpile_source(pine_source, filename=filename)
499 generated_cpp = transpile_result.require_success()
500 except TranspileFailure
as error:
504 diagnostics=transpile_result.diagnostics,
507 if transpile_result.input_kind_schema == 1
and legacy_input_kind_version(
508 request_identity[
"codegen"][
"version"]
512 "input_kind_schema: 1 contradicts recorded codegen version; "
513 "codegen >= 1.1.0 is required",
516 generated_cpp_sha256 = hashlib.sha256(
517 generated_cpp.encode(
"utf-8")
519 artifact_identity = {
520 "request_identity": request_identity,
521 "generated_cpp_sha256": generated_cpp_sha256,
523 artifact_key = _json_sha256(artifact_identity)
524 existing = self._load_artifact(
525 artifact_key=artifact_key,
526 request_key=request_key,
527 request_identity=request_identity,
528 output_extension=output_extension,
532 if existing
is not None:
534 raise _wrapper_bypass_error(
"artifact_cache")
536 existing = self._validate_built_plugin_locked(existing)
537 self._write_request_index(request_key, artifact_key)
540 artifact = self._compile_and_publish(
541 artifact_key=artifact_key,
542 request_key=request_key,
543 request_identity=request_identity,
544 output_extension=output_extension,
545 generated_cpp=generated_cpp,
546 generated_cpp_sha256=generated_cpp_sha256,
547 inputs=transpile_result.inputs,
548 input_kind_schema=transpile_result.input_kind_schema,
549 strategy_params=transpile_result.strategy_params,
550 diagnostics=transpile_result.diagnostics,
551 request_feed_inventory=transpile_result.request_feed_inventory,
552 source_name=filename,
557 self._write_request_index(request_key, artifact_key)
562 generated_cpp: bytes,
564 expected_sha256: str,
565 inputs: Sequence[Mapping[str, Any]],
566 input_kind_schema: int |
None,
567 strategy_params: Mapping[str, Any],
568 request_feed_inventory: Mapping[str, Any] |
None,
569 diagnostics: Sequence[TranspileDiagnostic] = (),
570 filename: str =
"<input>",
571 ) -> StrategyArtifact:
572 """Compile checked UTF-8 C++ bytes with the caller's emission metadata.
574 Verify the raw bytes before resolving a compiler or publishing anything.
575 This entry never transpiles and never returns before a real compile.
576 Supply metadata from the checked emission; no inventory is inferred.
579 if not isinstance(generated_cpp, bytes):
580 raise _checked_cpp_error(
"bytes",
"generated_cpp must be raw bytes")
582 not isinstance(expected_sha256, str)
583 or re.fullmatch(
r"[0-9a-fA-F]{64}", expected_sha256)
is None
585 raise _checked_cpp_error(
586 "expected_sha256",
"expected_sha256 must contain exactly 64 hex digits"
588 actual_sha256 = hashlib.sha256(generated_cpp).hexdigest()
589 if actual_sha256 != expected_sha256.lower():
591 "generated_cpp_hash",
592 "checked generated C++ SHA-256 mismatch: "
593 f
"expected {expected_sha256}, actual {actual_sha256}",
594 actual_sha256=actual_sha256,
595 expected_sha256=expected_sha256,
598 generated_text = generated_cpp.decode(
"utf-8")
599 except UnicodeDecodeError
as error:
600 raise _checked_cpp_error(
601 "utf8", f
"checked generated C++ is not strict UTF-8: {error}"
604 self._check_build_configuration()
607 _require_wrapper_codegen_identity()
609 else codegen_identity()
611 except ModuleNotFoundError
as error:
614 "checked C++ requires its emitting codegen installation",
617 not isinstance(inputs, (list, tuple))
618 or any(
not isinstance(item, Mapping)
for item
in inputs)
619 or not isinstance(strategy_params, Mapping)
621 input_kind_schema
is None
622 or (type(input_kind_schema)
is int
and input_kind_schema == 1)
624 or not isinstance(filename, str)
626 request_feed_inventory
is not None
628 not isinstance(request_feed_inventory, Mapping)
629 or request_feed_inventory.get(
"schema")
630 != REQUEST_FEED_INVENTORY_SCHEMA
631 or request_feed_inventory.get(
"source_sha256") != actual_sha256
634 or (request_feed_inventory
is None and codegen_supports_request_inventory())
636 raise _checked_cpp_error(
638 "checked emission requires its real inputs, input-kind schema, "
639 "strategy parameters and source-matching request-feed inventory "
640 "(when the loaded producer supports one)",
642 if not isinstance(diagnostics, (list, tuple))
or any(
643 not isinstance(item, TranspileDiagnostic)
for item
in diagnostics
645 raise _checked_cpp_error(
646 "diagnostics",
"checked diagnostics must be TranspileDiagnostic objects"
648 if any(item.severity ==
"error" for item
in diagnostics):
651 "checked emission has error diagnostics",
652 diagnostics=diagnostics,
654 if input_kind_schema == 1
and legacy_input_kind_version(codegen.version):
655 raise _checked_cpp_error(
656 "metadata",
"input_kind_schema: 1 requires codegen >= 1.1.0"
660 emission = copy.deepcopy(
662 "inputs": [dict(item)
for item
in inputs],
663 "input_kind_schema": input_kind_schema,
664 "strategy_params": dict(strategy_params),
665 "request_feed_inventory": (
666 dict(request_feed_inventory)
667 if request_feed_inventory
is not None
670 "diagnostics": [item.to_dict()
for item
in diagnostics],
671 "source_name": filename,
674 emission_sha256 = _json_sha256(emission)
675 except (TypeError, ValueError)
as error:
676 raise _checked_cpp_error(
678 f
"checked emission metadata is not JSON-serializable: {error}",
680 engine = self._resolve_engine_layout()
681 compiler = self._resolve_compiler_identity()
682 eigen = self._resolve_eigen_identity(engine)
683 output_extension, link_mode = self._platform_link_mode()
684 request_identity = self._request_identity(
685 source_sha256=actual_sha256,
690 output_extension=output_extension,
693 REQUEST_FEED_INVENTORY_SCHEMA
694 if request_feed_inventory
is not None
698 request_identity[
"source_kind"] =
"checked_cpp"
699 request_identity[
"emission_metadata_sha256"] = emission_sha256
700 request_key = _json_sha256(request_identity)
701 artifact_key = _json_sha256(
703 "request_identity": request_identity,
704 "generated_cpp_sha256": actual_sha256,
707 lock_path = self.
cache_dir /
"locks" / f
"{request_key}.lock"
709 artifact = self._compile_and_publish(
710 artifact_key=artifact_key,
711 request_key=request_key,
712 request_identity=request_identity,
713 output_extension=output_extension,
714 generated_cpp=generated_text,
715 generated_cpp_sha256=actual_sha256,
716 inputs=emission[
"inputs"],
717 input_kind_schema=input_kind_schema,
718 strategy_params=emission[
"strategy_params"],
719 diagnostics=diagnostics,
720 request_feed_inventory=emission[
"request_feed_inventory"],
721 source_name=filename,
726 self._write_request_index(request_key, artifact_key)
771 """Load and validate a pending source-cache plugin in the calling process.
773 Call this in a fresh process to separate the first plugin load from
774 compilation. This method starts no process and needs neither codegen nor
775 a local engine/compiler installation. It checks the recorded engine ABI
776 and content hashes under the request lock before publishing success.
777 Already validated artifacts are checked without loading or rewriting.
778 The caller supplies process isolation; this method does not install a jail.
783 keys = (artifact.request_key, artifact.artifact_key)
786 or any(character
not in "0123456789abcdef" for character
in key)
790 "cache",
"validation requires valid source-cache keys"
792 artifact_dir = self.
cache_dir /
"artifacts" / artifact.artifact_key
793 output_extension, _ = self._platform_link_mode()
795 artifact_dir /
"manifest.json",
796 artifact_dir /
"provenance.json",
797 artifact_dir /
"generated.cpp",
798 artifact_dir / f
"strategy{output_extension}",
801 artifact.manifest_path,
802 artifact.provenance_path,
803 artifact.generated_cpp_path,
804 artifact.plugin_path,
806 if actual_paths != expected_paths
or any(
807 path.resolve().parent != artifact_dir
for path
in actual_paths
810 "cache",
"only this builder's source-cache artifacts may be validated"
812 lock_path = self.
cache_dir /
"locks" / f
"{artifact.request_key}.lock"
814 return self._validate_built_plugin_locked(artifact)
887 artifact: StrategyArtifact,
889 explicit_path: str | os.PathLike[str] |
None =
None,
891 """Admit a source-cache artifact for selected execution under its request lock.
893 Ordinary builds do not call this method. An existing block is re-resolved
894 and verified without changing its expected hash. An interrupted two-file
895 enrichment leaves selected admission refused until enrichment is re-run.
896 A retry repairs missing or stale metadata, including a stale request_identity,
897 from the validating side under the same lock. A validated old cache with a
898 matching local engine may acquire metadata, never new plugin bytes or keys.
899 An explicit path also supports the non-derivable third archive layout.
900 Arbitrary precompiled artifacts cannot acquire an identity here.
903 keys = (artifact.request_key, artifact.artifact_key)
906 or any(character
not in "0123456789abcdef" for character
in key)
910 "window_plan_identity",
911 "enrichment requires valid source-cache keys",
913 artifact_dir = self.
cache_dir /
"artifacts" / artifact.artifact_key
914 output_extension, _ = self._platform_link_mode()
916 artifact_dir /
"manifest.json",
917 artifact_dir /
"provenance.json",
918 artifact_dir /
"generated.cpp",
919 artifact_dir / f
"strategy{output_extension}",
922 artifact.manifest_path,
923 artifact.provenance_path,
924 artifact.generated_cpp_path,
925 artifact.plugin_path,
927 if actual_paths != expected_paths
or any(
928 path.resolve().parent != artifact_dir
for path
in actual_paths
931 "window_plan_identity",
932 "only this builder's validated source-cache artifacts may be enriched",
934 lock_path = self.
cache_dir /
"locks" / f
"{artifact.request_key}.lock"
937 manifest = json.loads(
938 artifact.manifest_path.read_text(encoding=
"utf-8"),
940 provenance = json.loads(
941 artifact.provenance_path.read_text(encoding=
"utf-8")
943 except (OSError, ValueError)
as error:
945 "window_plan_manifest_invalid",
946 f
"cannot read cached manifest/provenance: {artifact_dir}: {error}",
948 if not isinstance(manifest, dict)
or not isinstance(provenance, dict):
950 "window_plan_manifest_invalid",
951 "source-cache manifest and provenance must be JSON objects",
955 originals = (dict(manifest), dict(provenance))
957 request_identity = manifest.get(
"request_identity")
958 if "window_plan" in manifest
or "window_plan" in provenance:
962 for location, document
in (
963 (
"manifest", manifest),
964 (
"provenance", provenance),
966 if "window_plan" not in document:
969 identity = WindowPlanIdentity.from_dict(document[
"window_plan"])
970 _check_compile_identity(
972 document.get(
"request_identity"),
975 except WindowPlanError:
977 candidates.append((identity, document))
980 "window_plan_manifest_invalid",
981 "no cached window_plan block validates against its "
982 "file's request_identity",
984 identity = candidates[0][0]
985 if any(candidate != identity
for candidate, _
in candidates[1:]):
987 "window_plan_identity",
988 "manifest.window_plan differs from provenance.window_plan",
990 block = identity.to_dict()
994 request_identity = next(
996 document[
"request_identity"]
997 for _, document
in candidates
998 if _json_sha256(document[
"request_identity"])
999 == artifact.request_key
1004 not isinstance(request_identity, Mapping)
1005 or _json_sha256(request_identity) != artifact.request_key
1008 "window_plan_identity",
1009 "source-cache request_identity differs from its request_key",
1011 if block
is not None:
1012 for document
in (manifest, provenance):
1013 document[
"window_plan"] = block
1014 if document.get(
"request_identity") != request_identity:
1015 document[
"request_identity"] = request_identity
1016 if document
is provenance:
1017 document[
"compile"] = request_identity[
"compile"]
1019 cached = self._load_artifact(
1020 artifact_key=artifact.artifact_key,
1021 request_key=artifact.request_key,
1022 request_identity=request_identity,
1023 output_extension=output_extension,
1025 metadata=(manifest, provenance),
1026 allow_deferred=
True,
1031 f
"cannot enrich an invalid source-cache artifact: {artifact_dir}",
1033 if manifest[
"plugin_validation"] == {
"status":
"deferred"}:
1035 "plugin_validation",
1036 "artifact plugin validation is pending; "
1037 "call validate_built_plugin first",
1040 engine = self._resolve_engine_layout()
1041 except (ArtifactBuildError, OSError, RuntimeError, ValueError)
as error:
1043 "window_plan_identity",
1044 f
"cannot verify local engine identity: {self.engine_root}: {error}",
1046 _require_engine_layout_identity(engine, request_identity)
1047 if block
is not None:
1048 path = resolve_window_plan_path(explicit_path, self.
engine_root, block)
1049 verify_window_plan(block, path)
1050 elif explicit_path
is not None:
1051 block = _associate_window_plan_path(engine, explicit_path)
1053 block = associate_window_plan(engine)
1055 if derive_window_plan_path(engine)
is None:
1058 "the selected archive requires an explicit absolute "
1059 "window_plan path for source-cache enrichment",
1062 "window_plan_missing",
1063 "the selected engine archive has no window_plan helper sibling",
1065 manifest[
"window_plan"] = block
1066 provenance[
"window_plan"] = block
1067 require_window_plan_block(manifest, provenance)
1069 for destination, payload, original
in zip(
1070 (artifact.manifest_path, artifact.provenance_path),
1071 (manifest, provenance),
1076 if payload != original:
1077 self._write_window_plan_metadata(destination, payload)
1078 except OSError
as error:
1081 f
"cannot publish window_plan metadata: {artifact_dir}: {error}",