Parse SEC official release datelines with September abbreviations
This commit is contained in:
1 parent
655813d4d9
commit
9038ee2efa
2 files changed
+13
No files matched your search
@@ -69,6 +69,7 @@ def published(value: str) -> tuple[datetime, str]:
|
|||||||
if not isinstance(value, str):
|
if not isinstance(value, str):
|
||||||
raise ValueError('官方材料缺少可识别的发布时间')
|
raise ValueError('官方材料缺少可识别的发布时间')
|
||||||
value = value.strip().replace('\xa0', ' ')
|
value = value.strip().replace('\xa0', ' ')
|
||||||
|
value = re.sub(r'\bSept(?=\.? \d)', 'Sep', value)
|
||||||
if re.fullmatch(r'\d{4}-\d{2}-\d{2}', value):
|
if re.fullmatch(r'\d{4}-\d{2}-\d{2}', value):
|
||||||
return datetime.fromisoformat(value).replace(tzinfo=UTC), 'date'
|
return datetime.fromisoformat(value).replace(tzinfo=UTC), 'date'
|
||||||
try:
|
try:
|
||||||
@@ -120,6 +121,8 @@ def article(source: str, html: str, url: str, *, release_date: str | None = None
|
|||||||
if not title:
|
if not title:
|
||||||
raise ValueError('官方正文缺少标题')
|
raise ValueError('官方正文缺少标题')
|
||||||
values = sel.css('meta[property="article:published_time"]::attr(content), meta[name="DC.date.issued"]::attr(content), meta[name="dcterms.date"]::attr(content), time::attr(datetime)').getall()
|
values = sel.css('meta[property="article:published_time"]::attr(content), meta[name="DC.date.issued"]::attr(content), meta[name="dcterms.date"]::attr(content), time::attr(datetime)').getall()
|
||||||
|
if source == 'sec':
|
||||||
|
values += sel.css('.field--name-dynamic-twig-fieldnode-press-release-lead-in').xpath('normalize-space(.)').getall()
|
||||||
for script in sel.css('script[type="application/ld+json"]::text').getall():
|
for script in sel.css('script[type="application/ld+json"]::text').getall():
|
||||||
try:
|
try:
|
||||||
data = json.loads(script)
|
data = json.loads(script)
|
||||||
|
|||||||
@@ -1376,3 +1376,13 @@ def test_doj_captioned_photo_is_not_a_document_gap(trading):
|
|||||||
item, = asyncio.run(collect())
|
item, = asyncio.run(collect())
|
||||||
assert item['extra']['attachments'] == []
|
assert item['extra']['attachments'] == []
|
||||||
assert not obj.cursor.get('attachment_retries')
|
assert not obj.cursor.get('attachment_retries')
|
||||||
|
|
||||||
|
|
||||||
|
def test_sec_uses_official_dateline_not_last_reviewed_date():
|
||||||
|
body = '''<main><h1>SEC regulatory announcement</h1>
|
||||||
|
<div class="field--name-dynamic-twig-fieldnode-press-release-lead-in">Washington D.C., Sept. 30, 2026 —</div>
|
||||||
|
<div class="node-details-layout__main-region__content"><div class="field--name-body">The Commission proposes digital asset market rules for regulated financial institutions.</div></div>
|
||||||
|
<div class="date-modified">Last Reviewed or Updated: November 5, 2026</div></main>'''
|
||||||
|
item = article('sec', body, 'https://www.sec.gov/newsroom/press-releases/example')
|
||||||
|
assert item['published_at'] == datetime(2026, 9, 30, tzinfo=UTC)
|
||||||
|
assert item['extra']['published_at_precision'] == 'date'
|
||||||
Reference in new issue
Block a user