1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
| import urllib.request,urllib.parse,http.cookiejar,json,concurrent.futures,re
| import pdfplumber
| from pathlib import Path
| P=Path(__file__).parent/'standards';P.mkdir(exist_ok=True)
| opener=urllib.request.build_opener(urllib.request.HTTPCookieProcessor(http.cookiejar.CookieJar()))
| base='https://sppt.cfsa.net.cn:8086'
| data=urllib.parse.urlencode({'task':'index'}).encode()
| r=opener.open(base+'/db',data,timeout=25)
| b=r.read();(P/'cfsa-index.html').write_bytes(b)
| def search(keyword):
| data=urllib.parse.urlencode({'isLength':'9999','num_tn':'99','standard_type':'','keyword':keyword}).encode()
| r=opener.open(base+'/db?task=indexSearch',data,timeout=25)
| out=json.loads(r.read().decode('utf8'));return keyword,out
| keywords=['DL-苹果酸','乳酸链球菌素','木糖醇','乳酸钙','山梨糖醇','柠檬酸钙','柠檬酸锌','甘油','食用香精','食品用香精','甜菊糖苷','葡萄糖酸亚铁','赤藓糖醇','黄原胶','赖氨酸','燕窝','丹参','花生衣','山药','甜菜根','枸杞']
| with concurrent.futures.ThreadPoolExecutor(max_workers=4) as pool:results=dict(pool.map(search,keywords))
| (P/'official-metadata.json').write_text(json.dumps(results,ensure_ascii=False,indent=2),encoding='utf8')
| codes={'GB 25544-2010','GB 1886.231-2023','GB 1886.234-2016','GB 1886.21-2016','GB 1886.187-2016','GB 1903.14-2016','GB 1903.49-2020','GB 29950-2013','GB 30616-2020','GB 1886.355-2022','GB 1903.10-2015','GB 26404-2011','GB 1886.41-2025','GB 1903.1-2015'}
| files=[]
| for keyword,items in results.items():
| for x in items:
| if x['CODE'] not in codes:continue
| if any(f['metadata']['ID']==x['ID'] for f in files):continue
| if x['TABLENAME'] not in ['2','5']:continue
| for attachment in x['FJ']:
| data=urllib.parse.urlencode({'task':'d_p','file_guid':attachment['ID_F']}).encode()
| r=opener.open(base+'/cfsa_aiguo',data,timeout=30);pdf=r.read();assert pdf.startswith(b'%PDF'),x
| fn=re.sub('[^a-zA-Z0-9.-]+','_',x['CODE'])+('_amendment' if '修改单' in x['TITLE'] else '')+'.pdf'
| path=P/fn;path.write_bytes(pdf)
| with pdfplumber.open(path) as doc:
| texts=[p.extract_text() or '' for p in doc.pages]
| path.with_suffix('.txt').write_text('\n\n'.join('PAGE '+str(i+1)+'\n'+t for i,t in enumerate(texts)),encoding='utf8')
| files.append({'metadata':x,'file':str(path),'source_url':base+'/db?task=goto&type='+x['TABLENAME']+'&guid='+x['ID']})
| print(fn,'pages',len(texts),'characters',sum(map(len,texts)))
| (P/'downloaded-manifest.json').write_text(json.dumps(files,ensure_ascii=False,indent=2),encoding='utf8')
|
|
