import urllib.request,urllib.parse,http.cookiejar,json,concurrent.futures,re
import pdfplumber
from pathlib import Path
P=Path(__file__).parent/'standards';P.mkdir(exist_ok=True)
opener=urllib.request.build_opener(urllib.request.HTTPCookieProcessor(http.cookiejar.CookieJar()))
base='https://sppt.cfsa.net.cn:8086'
data=urllib.parse.urlencode({'task':'index'}).encode()
r=opener.open(base+'/db',data,timeout=25)
b=r.read();(P/'cfsa-index.html').write_bytes(b)
def search(keyword):
 data=urllib.parse.urlencode({'isLength':'9999','num_tn':'99','standard_type':'','keyword':keyword}).encode()
 r=opener.open(base+'/db?task=indexSearch',data,timeout=25)
 out=json.loads(r.read().decode('utf8'));return keyword,out
keywords=['DL-苹果酸','乳酸链球菌素','木糖醇','乳酸钙','山梨糖醇','柠檬酸钙','柠檬酸锌','甘油','食用香精','食品用香精','甜菊糖苷','葡萄糖酸亚铁','赤藓糖醇','黄原胶','赖氨酸','燕窝','丹参','花生衣','山药','甜菜根','枸杞']
with concurrent.futures.ThreadPoolExecutor(max_workers=4) as pool:results=dict(pool.map(search,keywords))
(P/'official-metadata.json').write_text(json.dumps(results,ensure_ascii=False,indent=2),encoding='utf8')
codes={'GB 25544-2010','GB 1886.231-2023','GB 1886.234-2016','GB 1886.21-2016','GB 1886.187-2016','GB 1903.14-2016','GB 1903.49-2020','GB 29950-2013','GB 30616-2020','GB 1886.355-2022','GB 1903.10-2015','GB 26404-2011','GB 1886.41-2025','GB 1903.1-2015'}
files=[]
for keyword,items in results.items():
 for x in items:
  if x['CODE'] not in codes:continue
  if any(f['metadata']['ID']==x['ID'] for f in files):continue
  if x['TABLENAME'] not in ['2','5']:continue
  for attachment in x['FJ']:
   data=urllib.parse.urlencode({'task':'d_p','file_guid':attachment['ID_F']}).encode()
   r=opener.open(base+'/cfsa_aiguo',data,timeout=30);pdf=r.read();assert pdf.startswith(b'%PDF'),x
   fn=re.sub('[^a-zA-Z0-9.-]+','_',x['CODE'])+('_amendment' if '修改单' in x['TITLE'] else '')+'.pdf'
   path=P/fn;path.write_bytes(pdf)
   with pdfplumber.open(path) as doc:
    texts=[p.extract_text() or '' for p in doc.pages]
   path.with_suffix('.txt').write_text('\n\n'.join('PAGE '+str(i+1)+'\n'+t for i,t in enumerate(texts)),encoding='utf8')
   files.append({'metadata':x,'file':str(path),'source_url':base+'/db?task=goto&type='+x['TABLENAME']+'&guid='+x['ID']})
   print(fn,'pages',len(texts),'characters',sum(map(len,texts)))
(P/'downloaded-manifest.json').write_text(json.dumps(files,ensure_ascii=False,indent=2),encoding='utf8')