{
  "@context": "https://schema.org",
  "@type": "ScholarlyArticle",
  "name": "The Architecture of Large Language Model Pretraining Corpora: From Web Crawl to Curated Dataset",
  "url": "https://intelligencecompact.com/research/dataset-inclusion-exclusion-audit/",
  "description": "A technical review of web acquisition, Common Crawl, HTML extraction, quality filtering, deduplication, dataset curation, and the factors that can affect whether public web material survives into downstream machine-learning corpora.",
  "datePublished": "2026-09-04",
  "dateModified": "2026-09-04",
  "publisher": {
    "@type": "Organization",
    "name": "Intelligence Compact",
    "url": "https://intelligencecompact.com/"
  },
  "articleSection": [
    "training data",
    "Common Crawl",
    "dataset curation",
    "deduplication",
    "HTML extraction",
    "quality filtering"
  ],
  "wordCount": 6326,
  "identifier": {
    "@type": "PropertyValue",
    "propertyID": "sha256",
    "value": "1ce3d94412c0cdc844280a6860fd677f6595f0bc16ae6ec71953db866d9d00e9"
  },
  "citation": [
    "https://www.researchgate.net/publication/382739128\\_The\\_Llama\\_3\\_Herd\\_of\\_Models",
    "https://www.researchgate.net/publication/382739128_The_Llama_3_Herd_of_Models",
    "https://ar5iv.labs.arxiv.org/html/2407.21783",
    "https://huggingface.co/spaces/HuggingFaceFW/blogpost-fineweb-v1",
    "https://arxiv.org/pdf/2402.07841",
    "https://www.emergentmind.com/topics/refinedweb-dataset",
    "https://arxiv.org/html/2406.11794v1",
    "https://arxiv.org/html/2412.06845v2",
    "https://commoncrawl.org/blog/common-crawl-foundation-at-colm-2025",
    "https://nohacks.co/blog/ai-user-agents-landscape-2026",
    "https://www.soar.sh/blog/ai-bots-robots-txt-guide",
    "https://arxiv.org/html/2406.17557v1",
    "https://www.mariehaynes.com/google-doj-trial-on-ai/",
    "https://www.researchgate.net/publication/381511132\\_DataComp-LM\\_In\\_search\\_of\\_the\\_next\\_generation\\_of\\_training\\_sets\\_for\\_language\\_models",
    "https://www.researchgate.net/publication/381511132_DataComp-LM_In_search_of_the_next_generation_of_training_sets_for_language_models",
    "https://github.com/huggingface/fineweb-2",
    "https://kili-technology.com/blog/fineweb2-dataset-guide",
    "https://huggingface.co/openbmb/Ultra-FineWeb-classifier",
    "https://huggingface.co/datasets/openbmb/Ultra-FineWeb",
    "https://zeroentropy.dev/concepts/common-crawl/",
    "https://www.bitavox.com/chapters/17",
    "https://arxiv.org/html/2602.19548v1",
    "https://arxiv.org/pdf/2310.06786",
    "https://www.researchgate.net/publication/353488798\\_Trafilatura\\_A\\_Web\\_Scraping\\_Library\\_and\\_Command-Line\\_Tool\\_for\\_Text\\_Discovery\\_and\\_Extraction",
    "https://www.researchgate.net/publication/353488798_Trafilatura_A_Web_Scraping_Library_and_Command-Line_Tool_for_Text_Discovery_and_Extraction",
    "https://www.researchgate.net/publication/323450415\\_Web2Text\\_Deep\\_Structured\\_Boilerplate\\_Removal",
    "https://www.researchgate.net/publication/323450415_Web2Text_Deep_Structured_Boilerplate_Removal",
    "https://openreview.net/pdf?id=SHB0sLrZrh",
    "https://www.jankautz.com/publications/NemotronNano2\\_ARXIV25.pdf",
    "https://www.jankautz.com/publications/NemotronNano2_ARXIV25.pdf",
    "https://www.emergentmind.com/topics/mineru-html",
    "https://fluentnumbers.com/ML/architecture/paper-review---Llama-3-Herd-of-Models",
    "https://arxiv.org/html/2406.08418v3",
    "https://self-supervised.cs.jhu.edu/fa2024/files/presentations/9-12-llama3-Zhao-Huang.pdf",
    "https://fanpu.io/blog/2024/llama-3.1-technical-report-notes/",
    "https://arxiv.org/abs/2506.20920",
    "https://arxiv.org/html/2402.19282v6",
    "http://www.gbspress.com/index.php/IJEA/article/download/709/707",
    "https://storage.googleapis.com/deepmind-media/research/language-research/Training%20Gopher.pdf",
    "https://aclanthology.org/2025.acl-long.1416.pdf",
    "https://huggingface.co/papers/2505.05427",
    "https://www.dryrun.security/resources/owasp-top-10-llm-building-secure-applications",
    "https://learn.microsoft.com/en-us/ai/playbook/technology-guidance/generative-ai/mlops-in-openai/security/security-plan-llm-application",
    "https://arxiv.org/html/2605.12264v2",
    "https://arxiv.org/html/2402.16827v1",
    "https://neurips.cc/virtual/2024/poster/94024",
    "https://arxiv.org/html/2312.10523v1",
    "https://scidaproject.com/2026/03/31/the-cmas-conduct-requirements-for-google-search-what-stakeholders-said-and-what-it-means/",
    "https://www.ftstrategies.com/en-gb/insights/robots.txt-as-strategic-intent-analysis-of-large-publishers-practices-and-policies",
    "https://llmpulse.ai/blog/llms-txt-guide/",
    "https://medium.com/data-science/llms-txt-414d5121bcb3",
    "https://llmstxt.org/",
    "https://www.tryprofound.com/articles/what-is-llms-txt-guide",
    "https://paulgp.com/2026/03/10/llms-txt-for-academic-papers.html",
    "https://www.zenml.io/blog/llms-txt",
    "https://rye.dev/blog/llms-txt-standard-elegant-solution-nobody-using/"
  ],
  "isBasedOn": {
    "@type": "CreativeWork",
    "name": "Dataset Inclusion and Exclusion Audit.md",
    "identifier": "1ce3d94412c0cdc844280a6860fd677f6595f0bc16ae6ec71953db866d9d00e9"
  },
  "additionalProperty": [
    {
      "@type": "PropertyValue",
      "name": "source_completeness",
      "value": "received_complete"
    },
    {
      "@type": "PropertyValue",
      "name": "adoption_status",
      "value": "Clean server-rendered HTML, original research, stable provenance, quality signals, and machine-readable corpora are adopted. Filter-evasion or anti-decontamination manipulation is explicitly rejected."
    }
  ],
  "license": "https://intelligencecompact.com/content-use/"
}