Parse SEC official release datelines with September abbreviations
Development CI / check-and-build (push) Successful in 2m16s
GitHub Git mirror / mirror (push) Successful in 6s
Development CI / deploy-development (push) Successful in 2m31s

This commit is contained in:
eezd committed 2026-11-05 13:43:06 +08:00
1 parent 655813d4d9
commit 9038ee2efa
2 files changed
+13

No files matched your search

+3
View File
@@ -69,6 +69,7 @@ def published(value: str) -> tuple[datetime, str]:
if not isinstance(value, str): if not isinstance(value, str):
raise ValueError('官方材料缺少可识别的发布时间') raise ValueError('官方材料缺少可识别的发布时间')
value = value.strip().replace('\xa0', ' ') value = value.strip().replace('\xa0', ' ')
value = re.sub(r'\bSept(?=\.? \d)', 'Sep', value)
if re.fullmatch(r'\d{4}-\d{2}-\d{2}', value): if re.fullmatch(r'\d{4}-\d{2}-\d{2}', value):
return datetime.fromisoformat(value).replace(tzinfo=UTC), 'date' return datetime.fromisoformat(value).replace(tzinfo=UTC), 'date'
try: try:
@@ -120,6 +121,8 @@ def article(source: str, html: str, url: str, *, release_date: str | None = None
if not title: if not title:
raise ValueError('官方正文缺少标题') raise ValueError('官方正文缺少标题')
values = sel.css('meta[property="article:published_time"]::attr(content), meta[name="DC.date.issued"]::attr(content), meta[name="dcterms.date"]::attr(content), time::attr(datetime)').getall() values = sel.css('meta[property="article:published_time"]::attr(content), meta[name="DC.date.issued"]::attr(content), meta[name="dcterms.date"]::attr(content), time::attr(datetime)').getall()
if source == 'sec':
values += sel.css('.field--name-dynamic-twig-fieldnode-press-release-lead-in').xpath('normalize-space(.)').getall()
for script in sel.css('script[type="application/ld+json"]::text').getall(): for script in sel.css('script[type="application/ld+json"]::text').getall():
try: try:
data = json.loads(script) data = json.loads(script)
+10
View File
@@ -1376,3 +1376,13 @@ def test_doj_captioned_photo_is_not_a_document_gap(trading):
item, = asyncio.run(collect()) item, = asyncio.run(collect())
assert item['extra']['attachments'] == [] assert item['extra']['attachments'] == []
assert not obj.cursor.get('attachment_retries') assert not obj.cursor.get('attachment_retries')
def test_sec_uses_official_dateline_not_last_reviewed_date():
body = '''<main><h1>SEC regulatory announcement</h1>
<div class="field--name-dynamic-twig-fieldnode-press-release-lead-in">Washington D.C., Sept. 30, 2026 —</div>
<div class="node-details-layout__main-region__content"><div class="field--name-body">The Commission proposes digital asset market rules for regulated financial institutions.</div></div>
<div class="date-modified">Last Reviewed or Updated: November 5, 2026</div></main>'''
item = article('sec', body, 'https://www.sec.gov/newsroom/press-releases/example')
assert item['published_at'] == datetime(2026, 9, 30, tzinfo=UTC)
assert item['extra']['published_at_precision'] == 'date'