{
  "schema_version": 1,
  "release_status": "draft-not-approved-for-publication",
  "protocol": "jpt-pairs-v1",
  "review_status": "pending-independent-review",
  "results_sha256": "9f57baf3fdfcc03dc9e059fe25660e67b5033f360a31c700eeeabef00ac58c64",
  "domain_documents": {
    "ai": 431,
    "literature": 416,
    "music": 465,
    "politics": 650,
    "science": 543
  },
  "systems": {
    "graphiti-0.31.2-codex-gpt-6.1-sol": {
      "mean_domain_f1": 75.65923575300937,
      "predictions_sha256": "4cbf4ed0049ad07dc3eca877f33d952ea912fcb81c11bb60101b05fab29865e3",
      "evidence": "retained local inference predictions rescored locally",
      "configuration": {
        "graphiti_core": "0.31.2",
        "openai_sdk": "2.54.0",
        "chat_model": "gpt-6.1-sol",
        "chat_controls": "provider_defaults",
        "retry_policy": "single_attempt_stop_on_any_error",
        "schema": "original-train-names-including-misc",
        "extraction_source_head": "50bec31822367bd3da02fa77fc2a44e012ea196f",
        "bridge_revision": "d745731e90c9cfefbb5591ee6091e60502e7521e",
        "immutable_provider_checkpoint": "not available; served model identifier only"
      }
    },
    "jpt-4b-author-released-2026-11-02": {
      "mean_domain_f1": 75.25997091991715,
      "predictions_sha256": null,
      "evidence": "Author-released current downloadable cache, rescored locally; predictions are covered by jpt-archive SHA256 and consumed member pins in the original source lock; historical paper checkpoint identity unconfirmed."
    },
    "gliner-large-v2.1-local": {
      "mean_domain_f1": 66.70263995503936,
      "predictions_sha256": "01e767b4ce9cf2123e64c6ce4901cdd2b1f484738bb4032dbef1d99b5115867a",
      "evidence": "retained local inference predictions rescored locally",
      "configuration": {
        "checkpoint": "urchade/gliner_large-v2.1",
        "revision": "abd49a1f1ebc12af1be84d06f6848221cf96dcad",
        "backbone": "microsoft/deberta-v3-large",
        "backbone_revision": "64a8c8eab3e352a784c658aef62be1662607476f",
        "package": "0.2.30",
        "threshold": 0.5,
        "flat_ner": true,
        "multi_label": false,
        "device": "cuda",
        "precision": "float32",
        "batch_size": 1,
        "max_words": 384,
        "max_encoder_tokens": 512,
        "seed": 0,
        "labels": "original-train-names-including-misc",
        "system": "gliner-large-v2.1/0.2.30/fp32/defaults/original-schema-v1"
      },
      "configuration_sha256": "c4deeca31bb7d8d4a1fbe603927c0074364b00b7e8983a3a98b3e886a5755c06",
      "assets_manifest_sha256": "3d907f97c3486d861b1b6cc03914735deaa8b6e5e4d8cfc8b99d27796dbe3783"
    },
    "modern-gliner-bi-base-v1.0-local": {
      "mean_domain_f1": 64.08261617266666,
      "predictions_sha256": "7ce6fd4c875d40db62d230299f05de443f014a98cbe3822bcf66e398705f56c2",
      "evidence": "Complete retained local GPU predictions, scored with the unchanged released JPT normalized-pair evaluator. No new inference for this publication.",
      "configuration": {
        "checkpoint": "knowledgator/modern-gliner-bi-base-v1.0",
        "revision": "2d73e4c02b52f96e2926baffc67b5e78532fc39c",
        "packages": {
          "gliner": "0.2.30",
          "torch": "2.6.0+cu124",
          "transformers": "4.51.3",
          "huggingface-hub": "0.36.2"
        },
        "threshold": 0.5,
        "flat_ner": true,
        "multi_label": false,
        "device": "cuda",
        "precision": "float32",
        "batch_size": 1,
        "seed": 0,
        "schema": "original-train-names-including-misc",
        "decoder": "native-flat",
        "inference_source_head": "17a3040a23077fdbe374078a22d47d0f41426a94",
        "inference_source_dirty": false,
        "assets_manifest_sha256": "58fd11919cffe927835fd670168297d1abf7d0ef0ab14f02d9b22c02a40fd6dd",
        "evaluation_date": "2026-11-03",
        "elapsed_seconds": 99.96800000000076,
        "peak_gpu_bytes": 901172224,
        "timing_scope": "Shared RTX3060Ti; loading/preflight included, not isolated throughput.",
        "training_overlap": "Not excluded; no uncontaminated zero-shot claim.",
        "review_summary": "Implementation and recorded evidence independently reviewed 2026-11-07 at f0c57ecdcf6cfd617c00cf6785cf75e55b874023; no independent GPU rerun or general accuracy guarantee.",
        "source_evidence": {
          "repository": "amadou-6e/agent-anvil",
          "revision": "d4a44504e709f3167358b9bf3a463ef456b80534",
          "path": "tools/llms/entity-extraction/docs/benchmark/evidence/local-candidates.json",
          "sha256": "748158f2191f380039a27fd0102feff31013bee781004d5ed3d7eda0bcc0a3a1"
        },
        "compatibility_note": "Checkpoint tokenizer rechecked; predictions identical to original retained run. Configuration and timing describe the tokenizer recheck at inference revision 17a3040; execution is the original scoring receipt at 1400727 for byte-identical predictions (same SHA256)."
      },
      "execution": {
        "source_head": "1400727ae15ba2967d7db36df31c3da9182f056a",
        "source_dirty": false,
        "input_sha256": "da3f74f360ca5690a8447287a65290f8771bcb0dffe3b88beca69ed490302079",
        "source_lock_sha256": "475e47870af4132776833e8d3a7e46e157cd8f8da9647f699ee0ead4415b2274",
        "mapping_sha256": "e9e0735430311a6b8ec6aed18ff0b7fbbb32cdf75bd65520095119bb287fc3f3"
      }
    },
    "gliner2.5-base-v1-local": {
      "mean_domain_f1": 60.54899655020154,
      "predictions_sha256": "3986292fd6471f44fbacf8f18dbbddcf93ee77511b9f581b5e0b3c36dd267eb2",
      "evidence": "Complete retained local GPU predictions, scored with the unchanged released JPT normalized-pair evaluator. No new inference for this publication.",
      "configuration": {
        "checkpoint": "fastino/gliner2.5-base-v1",
        "revision": "54785d51df8d86a0d8f2eb212fec15871aad2c4d",
        "packages": {
          "gliner2": "2.1.0",
          "torch": "2.6.0+cu124",
          "transformers": "5.8.0",
          "huggingface-hub": "1.33.0"
        },
        "threshold": 0.5,
        "device": "cuda",
        "precision": "float32",
        "batch_size": 1,
        "seed": 0,
        "schema": "original-train-names-including-misc",
        "decoder": "native-flat",
        "inference_source_head": "1400727ae15ba2967d7db36df31c3da9182f056a",
        "inference_source_dirty": false,
        "assets_manifest_sha256": "2ec9e75cfaf247b89b41ced66ae1866025fbfb788d1a00102767d5954c679c5f",
        "evaluation_date": "2026-11-03",
        "elapsed_seconds": 80.48500000000058,
        "peak_gpu_bytes": 812482048,
        "timing_scope": "Shared RTX3060Ti; loading/preflight included, not isolated throughput.",
        "training_overlap": "Not excluded; no uncontaminated zero-shot claim.",
        "review_summary": "Implementation and recorded evidence independently reviewed 2026-11-07 at f0c57ecdcf6cfd617c00cf6785cf75e55b874023; no independent GPU rerun or general accuracy guarantee.",
        "source_evidence": {
          "repository": "amadou-6e/agent-anvil",
          "revision": "d4a44504e709f3167358b9bf3a463ef456b80534",
          "path": "tools/llms/entity-extraction/docs/benchmark/evidence/local-candidates.json",
          "sha256": "748158f2191f380039a27fd0102feff31013bee781004d5ed3d7eda0bcc0a3a1"
        },
        "compatibility_note": "AutoExtractor boundary loader; direct checkpoint tokenizer.",
        "overlap_policy": "flat",
        "max_len": 4096,
        "include_spans": true
      },
      "execution": {
        "source_head": "1400727ae15ba2967d7db36df31c3da9182f056a",
        "source_dirty": true,
        "input_sha256": "da3f74f360ca5690a8447287a65290f8771bcb0dffe3b88beca69ed490302079",
        "source_lock_sha256": "475e47870af4132776833e8d3a7e46e157cd8f8da9647f699ee0ead4415b2274",
        "mapping_sha256": "e9e0735430311a6b8ec6aed18ff0b7fbbb32cdf75bd65520095119bb287fc3f3"
      }
    },
    "gliner-bi-large-v2.0-local": {
      "mean_domain_f1": 66.16207524413997,
      "predictions_sha256": "2ac0c1a491771cdb181bc3e675e7951ccf1155071540c0f91232dd21150557e3",
      "evidence": "Complete retained local GPU predictions, scored with the unchanged released JPT normalized-pair evaluator. No new inference for this publication.",
      "configuration": {
        "checkpoint": "knowledgator/gliner-bi-large-v2.0",
        "revision": "71eec1d5fe57684cd6669ad8a1e1e3e7401a9a55",
        "packages": {
          "gliner": "0.2.30",
          "torch": "2.6.0+cu124",
          "transformers": "4.51.3",
          "huggingface-hub": "0.36.2"
        },
        "threshold": 0.5,
        "flat_ner": true,
        "multi_label": false,
        "device": "cuda",
        "precision": "float32",
        "batch_size": 1,
        "seed": 0,
        "schema": "original-train-names-including-misc",
        "decoder": "native-flat",
        "inference_source_head": "05f34bf1870c64751ae813ac614f401a1dd63827",
        "inference_source_dirty": false,
        "assets_manifest_sha256": "002daf72a88a466f721a0be2909b811ae0734dc1f6d2f4bb60fd1e181ea7ed5c",
        "evaluation_date": "2026-11-03",
        "elapsed_seconds": 116.20300000000134,
        "peak_gpu_bytes": 2170597888,
        "timing_scope": "Shared RTX3060Ti; loading/preflight included, not isolated throughput.",
        "training_overlap": "Not excluded; no uncontaminated zero-shot claim.",
        "review_summary": "Implementation and recorded evidence independently reviewed 2026-11-07 at dbcba5ebb0f38dfd40f894f956420b47a359c2e1; no independent GPU rerun or general accuracy guarantee.",
        "source_evidence": {
          "repository": "amadou-6e/agent-anvil",
          "revision": "d4a44504e709f3167358b9bf3a463ef456b80534",
          "path": "tools/llms/entity-extraction/docs/benchmark/evidence/next-local-candidates.json",
          "sha256": "818f9e53ef4b5392be13f617f74acb6305137018b0897c028b58902af0aa745e"
        },
        "compatibility_note": "Fixed0.5threshold, not the card0.3example."
      },
      "execution": {
        "source_head": "17a3040a23077fdbe374078a22d47d0f41426a94",
        "source_dirty": false,
        "input_sha256": "da3f74f360ca5690a8447287a65290f8771bcb0dffe3b88beca69ed490302079",
        "source_lock_sha256": "475e47870af4132776833e8d3a7e46e157cd8f8da9647f699ee0ead4415b2274",
        "mapping_sha256": "e9e0735430311a6b8ec6aed18ff0b7fbbb32cdf75bd65520095119bb287fc3f3"
      }
    },
    "nuner-zero-local": {
      "mean_domain_f1": 65.73980353198243,
      "predictions_sha256": "eebf079e067c1820237a050d6c8cdf49de3ba51e75da95f74f90c11dcbb0cc7b",
      "evidence": "Complete retained local GPU predictions, scored with the unchanged released JPT normalized-pair evaluator. No new inference for this publication.",
      "configuration": {
        "checkpoint": "numind/NuNER_Zero",
        "revision": "c90187673f464518dca09f41689184ed6976242c",
        "packages": {
          "gliner": "0.2.30",
          "torch": "2.6.0+cu124",
          "transformers": "4.51.3",
          "huggingface-hub": "0.36.2"
        },
        "threshold": 0.5,
        "flat_ner": true,
        "multi_label": false,
        "device": "cuda",
        "precision": "float32",
        "batch_size": 1,
        "seed": 0,
        "schema": "original-train-names-including-misc",
        "decoder": "author-adjacent-same-label-token-merge",
        "inference_source_head": "ad4619e193c4aed800cd77a9d0be78c9d588b64d",
        "inference_source_dirty": false,
        "assets_manifest_sha256": "5955b58e4040a8c424b77e01d3dfa46f6babce1b6473ee013579e7faaa73ca59",
        "evaluation_date": "2026-11-03",
        "elapsed_seconds": 119.98399999999674,
        "peak_gpu_bytes": 1839356416,
        "timing_scope": "Shared RTX3060Ti; loading/preflight included, not isolated throughput.",
        "training_overlap": "Not excluded; no uncontaminated zero-shot claim.",
        "review_summary": "Implementation and recorded evidence independently reviewed 2026-11-07 at dbcba5ebb0f38dfd40f894f956420b47a359c2e1; no independent GPU rerun or general accuracy guarantee.",
        "source_evidence": {
          "repository": "amadou-6e/agent-anvil",
          "revision": "d4a44504e709f3167358b9bf3a463ef456b80534",
          "path": "tools/llms/entity-extraction/docs/benchmark/evidence/next-local-candidates.json",
          "sha256": "818f9e53ef4b5392be13f617f74acb6305137018b0897c028b58902af0aa745e"
        },
        "compatibility_note": "GLiNER0.2.30 with documented adjacent-token merge; distinct from card0.1.12 loader."
      },
      "execution": {
        "source_head": "44f08673062afcc91f96a8404589b61b9ac411e5",
        "source_dirty": false,
        "input_sha256": "da3f74f360ca5690a8447287a65290f8771bcb0dffe3b88beca69ed490302079",
        "source_lock_sha256": "475e47870af4132776833e8d3a7e46e157cd8f8da9647f699ee0ead4415b2274",
        "mapping_sha256": "e9e0735430311a6b8ec6aed18ff0b7fbbb32cdf75bd65520095119bb287fc3f3"
      }
    }
  },
  "execution": {
    "repository": "amadou-6e/agent-anvil",
    "source_head": "78ba4c2b032a2ffc4a03e80ff0f715c0111ffe74",
    "source_dirty": false,
    "input_sha256": "da3f74f360ca5690a8447287a65290f8771bcb0dffe3b88beca69ed490302079",
    "source_lock_sha256": "475e47870af4132776833e8d3a7e46e157cd8f8da9647f699ee0ead4415b2274",
    "mapping_sha256": "e9e0735430311a6b8ec6aed18ff0b7fbbb32cdf75bd65520095119bb287fc3f3",
    "new_inference_for_this_release": false,
    "reproducibility": "Original implementation and prediction artifacts currently private; aggregate validation only in this repository."
  },
  "upstream_artifacts": {
    "jpt-archive": {
      "filename": "jpt-benchmarks.zip",
      "sha256": "df7dacfabe26e7080da99103c3b0d09b70c08366138c3ae8da41495d836cad2b",
      "byte_count": 1586014,
      "source": "https://drive.google.com/file/d/1D8aKQfvJQ65ELrwW0oITvFogWy96szFW/view"
    },
    "jpt-scorer": {
      "filename": "jpt-evaluator-source.txt",
      "sha256": "d75050e5a086ca12301c8ebf000b3bae76fa3a419756d6aba375ceb0277c8794",
      "byte_count": 18054,
      "source": "https://witness-ai-jpt-ner.hf.space/ static evaluator download, retrieved 2026-11-02"
    }
  },
  "excluded_artifacts": [
    "source texts",
    "gold annotations",
    "raw predictions",
    "author cache",
    "upstream scorer code",
    "model weights"
  ],
  "license_status": "owner-selected CC-BY-4.0 for original compilation/docs only; upstream materials excluded",
  "scientific_review": {
    "verdict": "proceed-no-blockers-for-scoped-jpt-path",
    "reviewed_source_head": "453729cabfa91ba099f45dc707cacd3c36bf7463",
    "scope": "JPT pair scorer adapter, source alignment, schema projection, complete-domain aggregation and three rows only",
    "limits": [
      "reviewer did not execute code",
      "reviewer did not open raw gold/cache contents",
      "whole source PR105 not reviewed or approved",
      "not publication approval"
    ]
  },
  "destination": {
    "repo_type": "dataset",
    "repo_id": "Amadou98/crossner-extractor-comparison",
    "status": "owner-selected; not created or uploaded by this workspace"
  },
  "licenses": {
    "code": "MIT",
    "original_results_and_documentation": "CC-BY-4.0",
    "upstream_materials": "retain original rights; not covered by our license grant"
  },
  "review_status_scope": "Legacy CSV/manifest review_status remains pending-independent-review for compatibility with the unchanged public v0.1.0 aggregate validator and provisional-comparison admission. Scoped source implementation/evidence approval for added rows is recorded in each configuration.review_summary; it is not an independently rerun-inference badge or whole-benchmark scientific acceptance.",
  "publication_refresh": {
    "date": "2026-11-07",
    "scope": "Four reviewed local configurations added to existing three; scorer, alignment and old values unchanged.",
    "source_revision": "d4a44504e709f3167358b9bf3a463ef456b80534",
    "systems": 7,
    "rows": 35
  },
  "release_status_scope": "Legacy preparation-era release_status retained for historical schema compatibility. The owner authorized this existing public Space aggregate refresh; scientific benchmark status remains provisional, not unapproved for publication. See publication_refresh and per-system review_summary for current scoped status."
}