主页 / coding / 问题 / 79202026
Accepted
Brad
Asked: 2024-11-19 11:38:13 +0800 CST

为什么我的 JSON 文件代码片段可以工作，但完整的 JSON 加载却不行

  • 772

我有一个 JSON 文件片段，我将其规范化并使用 record_path 和其他列上的元参数，它工作正常，但当我加载整个文件时，我收到一个关键错误。下面是我为片段和完整的 json 下载文件加载编写的代码。

片段作品

import pandas as pd

data = {
    "meta": {
        "disclaimer": "Do not rely on openFDA to make decisions regarding medical care. While we make every effort to ensure that data is accurate, you should assume all results are unvalidated. We may limit or otherwise restrict your access to the API in line with our Terms of Service.",
        "terms": "https://open.fda.gov/terms/",
        "license": "https://open.fda.gov/license/",
        "last_updated": "2024-11-15",
        "results": {
            "skip": 0,
            "limit": 2,
            "total": 118943
        }
    },
    "results": [
        {
            "product_ndc": "73647-062",
            "generic_name": "MENTHOL, CAMPHOR",
            "labeler_name": "Just Brands LLC",
            "brand_name": "JUST CBD - CBD AND THC ULTRA RELIEF",
            "active_ingredients": [
                {
                    "name": "CAMPHOR (SYNTHETIC)",
                    "strength": "2 g/100g"
                },
                {
                    "name": "MENTHOL",
                    "strength": "6 g/100g"
                }
            ],
            "finished": True,
            "packaging": [
                {
                    "package_ndc": "73647-062-04",
                    "description": "113 g in 1 BOTTLE, PUMP (73647-062-04)",
                    "marketing_start_date": "20230314",
                    "sample": False
                }
            ],
            "listing_expiration_date": "20251231",
            "openfda": {
                "manufacturer_name": ["Just Brands LLC"],
                "spl_set_id": ["f664eb79-8897-3a49-e053-2995a90a37b4"],
                "is_original_packager": [True],
                "unii": ["5TJD82A1ET", "L7T10EIP3A"]
            },
            "marketing_category": "OTC MONOGRAPH DRUG",
            "dosage_form": "GEL",
            "spl_id": "16c906dd-6989-9a79-e063-6394a90afa71",
            "product_type": "HUMAN OTC DRUG",
            "route": ["TOPICAL"],
            "marketing_start_date": "20230314",
            "product_id": "73647-062_16c906dd-6989-9a79-e063-6394a90afa71",
            "application_number": "M017",
            "brand_name_base": "JUST CBD - CBD AND THC ULTRA RELIEF"
        },
        {
            "product_ndc": "0591-4039",
            "marketing_end_date": "20250930",
            "generic_name": "CLOBETASOL PROPIONATE",
            "labeler_name": "Actavis Pharma, Inc.",
            "brand_name": "CLOBETASOL PROPIONATE",
            "active_ingredients": [
                {
                    "name": "CLOBETASOL PROPIONATE",
                    "strength": ".05 g/mL"
                }
            ],
            "finished": True,
            "packaging": [
                {
                    "package_ndc": "0591-4039-46",
                    "description": "1 BOTTLE in 1 CARTON (0591-4039-46)  / 59 mL in 1 BOTTLE",
                    "marketing_start_date": "20150828",
                    "marketing_end_date": "20250930",
                    "sample": False
                },
                {
                    "package_ndc": "0591-4039-74",
                    "description": "1 BOTTLE in 1 CARTON (0591-4039-74)  / 125 mL in 1 BOTTLE",
                    "marketing_start_date": "20150828",
                    "marketing_end_date": "20250930",
                    "sample": False
                }
            ],
            "openfda": {
                "manufacturer_name": ["Actavis Pharma, Inc."],
                "rxcui": ["861512"],
                "spl_set_id": ["907e425a-720a-4180-b97c-9e25008a3658"],
                "is_original_packager": [True],
                "unii": ["779619577M"]
            },
            "marketing_category": "NDA AUTHORIZED GENERIC",
            "dosage_form": "SPRAY",
            "spl_id": "33a56b8b-a9a6-4287-bbf4-d68ad0c59e07",
            "product_type": "HUMAN PRESCRIPTION DRUG",
            "route": ["TOPICAL"],
            "marketing_start_date": "20150828",
            "product_id": "0591-4039_33a56b8b-a9a6-4287-bbf4-d68ad0c59e07",
            "application_number": "NDA021835",
            "brand_name_base": "CLOBETASOL PROPIONATE",
            "pharm_class": [
                "Corticosteroid Hormone Receptor Agonists [MoA]",
                "Corticosteroid [EPC]"
            ]
        }
    ]
}

packaging_data = pd.json_normalize(
    data['results'], 
    record_path=["packaging"], 
    meta=['product_ndc', 'brand_name', 'generic_name']
)

active_ingredients_data = pd.json_normalize(
    data['results'], 
    record_path=["active_ingredients"], 
    meta=['product_ndc', 'brand_name', 'generic_name']
)

combined_data = pd.merge(
    packaging_data,
    active_ingredients_data,
    on=['product_ndc', 'brand_name', 'generic_name'],
    how='outer'
)

使用元数据时，JSON 文件的完整下载和加载会出现关键错误

import pandas as pd
import json
import requests, zipfile, io, os

cwd = os.getcwd()
zip_url = 'https://download.open.fda.gov/drug/ndc/drug-ndc-0001-of-0001.json.zip'
r = requests.get(zip_url)
z = zipfile.ZipFile(io.BytesIO(r.content))
z.extractall(cwd)

with open('drug-ndc-0001-of-0001.json', 'r') as file:
    data = json.load(file)

packaging_data = pd.json_normalize(
    data['results'], 
    record_path=["packaging"], 
    meta=['product_ndc', 'brand_name', 'generic_name']
)

active_ingredients_data = pd.json_normalize(
    data['results'], 
    record_path=["active_ingredients"], 
    meta=['product_ndc', 'brand_name', 'generic_name']
)

combined_data = pd.merge(
    packaging_data,
    active_ingredients_data,
    on=['product_ndc', 'brand_name', 'generic_name'],
    how='outer'
)
python

2 个回答

  1. Best Answer

    您的数据存在漏洞。有些记录没有“brand_name”，其他记录缺少用于扩展表格的“active_ingredients”列表。对于“brand_name”，您可以设置errors='ignore'填充默认值。如果您不喜欢该解决方案，您可以预先扫描列表并自行修改这些记录。

    “active_ingredients” 更成问题。该字段需要作为列表存在于所有记录中。您可以扫描这些记录并添加一个空列表或完全删除该记录，具体取决于您想如何解决问题。

    • 2

  2. 您需要允许缺少元键和“active_ingredients”。对于元键，我只会添加“n/a”（不可用），对于后一种情况，只需使用空列表即可。

    下载大量内容时，我更喜欢流式传输。

    因此，我建议：

    import requests
from zipfile import ZipFile
import json
from pathlib import Path
import pandas as pd

CHUNK = 4096
META = ["product_ndc", "brand_name", "generic_name"]
DEFAULTS = {
    "product_ndc": "n/a",
    "brand_name": "n/a",
    "generic_name": "n/a",
    "active_ingredients": []
}
url = "https://download.open.fda.gov/drug/ndc/drug-ndc-0001-of-0001.json.zip"
filename = Path(url.split("/")[-1])

with requests.get(url, stream=True) as response:
    response.raise_for_status()
    with filename.open("wb") as z:
        for chunk in response.iter_content(CHUNK):
            z.write(chunk)

with ZipFile(filename) as z:
    z.extractall()
    with Path(z.namelist()[0]).open() as j:
        data = json.load(j)
    for result in data["results"]:
        for key, value in DEFAULTS.items():
            result.setdefault(key, value)

packaging_data = pd.json_normalize(
    data["results"], record_path=["packaging"], meta=list(META) # call list() to satisfy mypy
)

active_ingredients_data = pd.json_normalize(
    data["results"], record_path=["active_ingredients"], meta=list(META) # call list() to satisfy mypy
)

combined_data = pd.merge(packaging_data, active_ingredients_data, on=META, how="outer")

print(combined_data.head())

    输出：

        package_ndc                                 description marketing_start_date sample marketing_end_date product_ndc brand_name   generic_name           name     strength
0  0002-0013-03   3 mL in 1 VIAL, MULTI-DOSE (0002-0013-03)            04-DEC-09    NaN                NaN   0002-0013        n/a  Insulin human  INSULIN HUMAN  100 [iU]/mL
1  0002-0013-10  10 mL in 1 VIAL, MULTI-DOSE (0002-0013-10)            27-JUN-83    NaN                NaN   0002-0013        n/a  Insulin human  INSULIN HUMAN  100 [iU]/mL
2  0002-0095-00            3 mL in 1 SYRINGE (0002-0095-00)            29-DEC-15    NaN                NaN   0002-0095        n/a  Insulin human  INSULIN HUMAN  500 [iU]/mL
3  0002-0096-00  20 mL in 1 VIAL, MULTI-DOSE (0002-0096-00)            06-JAN-97    NaN                NaN   0002-0096        n/a  Insulin human  INSULIN HUMAN  500 [iU]/mL
4  0002-0113-03   3 mL in 1 VIAL, MULTI-DOSE (0002-0113-03)            24-JUN-10    NaN                NaN   0002-0113        n/a  Insulin human  INSULIN HUMAN  100 [iU]/mL
    • 1

相关问题