my-home/zhixing-system
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

perf(sector_radar): narrow publication reads and index rank history

Browse Source
This commit is contained in:
yuxuanhui
2026-09-07 12:16:25 +08:00
parent e567e5f717
commit d7586b27f6
13 changed files with 1083 additions and 69 deletions
Download Patch File Download Diff File Expand all files Collapse all files
zhixing-server/src/zhixing_server/modules/sector_radar/application/details.py
+104 -66
View File
@@ -10,14 +10,13 @@ from decimal import Decimal
from ..domain.metrics import AmountNetStrategy, RatioTurnoverStrategy, SwingEqualThreeToTenStrategy
from ..domain.models import MetricKind, RadarPublication, RankedMetric, RankSide, SectorType
from ..domain.normalize import is_current_listed_stock
from ..domain.persistence import PublicationSourceGroup, SectorRadarRepository
from ..domain.persistence import HistoricalRanking, PublicationSourceGroup, SectorRadarRepository
from ..domain.source import (
DailyRow,
MoneyflowDcRow,
MoneyflowRow,
SectorIndexRow,
SectorMemberRow,
SourceSnapshot,
StockBasicRow,
TradeCalendarRow,
)
@@ -132,6 +131,39 @@ class SectorDetail:
similar_sectors: tuple[SimilarSector, ...]
class _RankHistory:
"""Index one day's selected ranks once, retaining complete versioned pool counts."""
def __init__(self, records: Sequence[HistoricalRanking]) -> None:
self.pools: dict[tuple[SectorType, MetricKind, str], int] = {}
self.rankings: dict[tuple[SectorType, str, MetricKind, str], RankedMetric] = {}
self.names: dict[tuple[SectorType, str], str] = {}
for record in records:
self.pools[(record.sector_type, record.metric_kind, record.metric_version)] = (
record.pool_size
)
if record.ranking is not None:
observation = record.ranking.observation
self.rankings.setdefault(
(
record.sector_type,
observation.sector_code,
record.metric_kind,
record.metric_version,
),
record.ranking,
)
self.names.setdefault(
(record.sector_type, observation.sector_code), observation.sector_name
)
def metric(self, sector_type: SectorType, sector_code: str, kind: MetricKind) -> HistoryMetric:
version = _METRIC_VERSIONS[kind]
size = self.pools.get((sector_type, kind, version), 0)
row = self.rankings.get((sector_type, sector_code, kind, version))
return _metric_from_ranking(row, size)
class ReadRadarDetails:
"""Reuse batched history and exact publication raw snapshots across read views."""
@@ -139,8 +171,8 @@ class ReadRadarDetails:
self.repository = repository
def history_data(
self, publication: RadarPublication
) -> tuple[dict[date, RadarPublication], dict[date, Sequence[RankedMetric]], tuple[date, ...]]:
self, publication: RadarPublication, sector_codes: Sequence[str]
) -> tuple[dict[date, RadarPublication], dict[date, _RankHistory], tuple[date, ...]]:
"""Load history in bounded batches; calendar holes remain explicit missing points."""
publications = {
item.target_trade_date: item
@@ -149,22 +181,23 @@ class ReadRadarDetails:
# Pin the current date to the response's chosen publication if a rebuild finishes
# during this request. All history rows are then fetched by these exact IDs.
publications[publication.target_trade_date] = publication
by_id = dict(
self.repository.load_publication_rankings(
tuple(item.publication_id for item in publications.values())
)
)
by_id: dict[str, list[HistoricalRanking]] = {}
for record in self.repository.load_ranked_history(
tuple(item.publication_id for item in publications.values()), sector_codes
):
by_id.setdefault(record.publication_id, []).append(record)
rows = {
day: by_id.get(item.publication_id, ())
day: _RankHistory(by_id.get(item.publication_id, ()))
if item.source_version == publication.source_version
else ()
else _RankHistory(())
for day, item in publications.items()
}
snapshots = self.snapshots(publication)
sources = self.repository.load_publication_rows(
publication.publication_id, (PublicationSourceGroup.CALENDAR,)
)
calendar = [
TradeCalendarRow.from_mapping(row)
for snapshot in snapshots.get(PublicationSourceGroup.CALENDAR, ())
for row in snapshot.rows
for row in sources.get(PublicationSourceGroup.CALENDAR, ())
]
dates = tuple(
sorted(
@@ -176,7 +209,7 @@ class ReadRadarDetails:
| set(publications)
)[-30:]
)
return publications, rows, dates
return publications, {day: rows.get(day, _RankHistory(())) for day in dates}, dates
def ranking_extras(
self, publication: RadarPublication, rankings: Sequence[RankedMetric], side: RankSide
@@ -184,16 +217,22 @@ class ReadRadarDetails:
"""Enrich one page from one batched thirty-session history, with no per-sector IO."""
if not rankings:
return {}
_, history, dates = self.history_data(publication)
snapshots = self.snapshots(publication)
_, history, dates = self.history_data(
publication, tuple(row.observation.sector_code for row in rankings)
)
sources = self.repository.load_publication_rows(
publication.publication_id,
(PublicationSourceGroup.CONCEPT_INDICES, PublicationSourceGroup.INDUSTRY_INDICES),
trade_date=publication.target_trade_date,
ts_codes=tuple(row.observation.sector_code for row in rankings),
)
indices = {
(item.sector_type, item.sector_code): item
for group, kind in (
(PublicationSourceGroup.CONCEPT_INDICES, SectorType.CONCEPT),
(PublicationSourceGroup.INDUSTRY_INDICES, SectorType.INDUSTRY),
)
for snapshot in snapshots.get(group, ())
for row in snapshot.rows
for row in sources.get(group, ())
for item in (SectorIndexRow.from_mapping(row, kind),)
if item.trade_date == publication.target_trade_date
}
@@ -202,12 +241,10 @@ class ReadRadarDetails:
observation = ranking.observation
key = (observation.sector_type, observation.sector_code)
index = indices.get(key)
current = history.get(publication.target_trade_date, ())
amount = metric_at(current, *key, MetricKind.AMOUNT).metric_value
ratio = metric_at(current, *key, MetricKind.RATIO).metric_value
points = [
metric_at(history.get(day, ()), *key, observation.metric_kind) for day in dates
]
current = history[publication.target_trade_date]
amount = current.metric(*key, MetricKind.AMOUNT).metric_value
ratio = current.metric(*key, MetricKind.RATIO).metric_value
points = [history[day].metric(*key, observation.metric_kind) for day in dates]
available_days = sum(not point.missing for point in points)
on_list_count = None
if available_days and side is not RankSide.ALL:
@@ -223,15 +260,6 @@ class ReadRadarDetails:
)
return result
def snapshots(
self, publication: RadarPublication
) -> dict[PublicationSourceGroup, list[SourceSnapshot]]:
"""Load exact source revisions, never global latest membership or provider data."""
grouped: dict[PublicationSourceGroup, list[SourceSnapshot]] = {}
for record in self.repository.load_publication_sources(publication.publication_id):
grouped.setdefault(record.source_group, []).append(record.snapshot)
return grouped
def history(self, target: date, sector_type: SectorType, sector_code: str) -> SectorHistory:
"""Return an exact-date publication and its compatible, past-only rank trajectory."""
publication = self.repository.get_successful_publication(target)
@@ -239,24 +267,15 @@ class ReadRadarDetails:
return SectorHistory(
"no_data", target, None, None, sector_type, sector_code, None, (), 0
)
publications, rows, dates = self.history_data(publication)
current_rows = rows.get(target, ())
name = next(
(
row.observation.sector_name
for row in current_rows
if row.observation.sector_type is sector_type
and row.observation.sector_code == sector_code
),
None,
)
publications, rows, dates = self.history_data(publication, (sector_code,))
name = rows[target].names.get((sector_type, sector_code))
points = tuple(
HistoryPoint(
day,
publications[day].publication_id if day in publications else None,
metric_at(rows.get(day, ()), sector_type, sector_code, MetricKind.AMOUNT),
metric_at(rows.get(day, ()), sector_type, sector_code, MetricKind.RATIO),
metric_at(rows.get(day, ()), sector_type, sector_code, MetricKind.SWING),
rows[day].metric(sector_type, sector_code, MetricKind.AMOUNT),
rows[day].metric(sector_type, sector_code, MetricKind.RATIO),
rows[day].metric(sector_type, sector_code, MetricKind.SWING),
)
for day in dates
)
@@ -286,15 +305,22 @@ class ReadRadarDetails:
latest = next((point for point in history.points if point.trade_date == target), None)
if latest:
summary = {kind.value: getattr(latest, kind.value) for kind in MetricKind}
snapshots = self.snapshots(history.publication)
sources = self.repository.load_publication_rows(
history.publication.publication_id,
(
PublicationSourceGroup.CONCEPT_INDICES,
PublicationSourceGroup.INDUSTRY_INDICES,
PublicationSourceGroup.STOCK_BASICS,
PublicationSourceGroup.MEMBERS,
),
)
indices = [
SectorIndexRow.from_mapping(row, kind)
for group, kind in (
(PublicationSourceGroup.CONCEPT_INDICES, SectorType.CONCEPT),
(PublicationSourceGroup.INDUSTRY_INDICES, SectorType.INDUSTRY),
)
for snapshot in snapshots.get(group, ())
for row in snapshot.rows
for row in sources.get(group, ())
]
index = next(
(
@@ -308,37 +334,44 @@ class ReadRadarDetails:
)
basics = {
item.ts_code: item
for snapshot in snapshots.get(PublicationSourceGroup.STOCK_BASICS, ())
for row in snapshot.rows
for row in sources.get(PublicationSourceGroup.STOCK_BASICS, ())
for item in (StockBasicRow.from_mapping(row),)
if is_current_listed_stock(item, target)
}
memberships: dict[str, dict[str, str]] = {}
for snapshot in snapshots.get(PublicationSourceGroup.MEMBERS, ()):
for row in snapshot.rows:
member = SectorMemberRow.from_mapping(row)
if member.trade_date == target and member.stock_code in basics:
memberships.setdefault(member.sector_code, {})[member.stock_code] = (
member.stock_name
)
for row in sources.get(PublicationSourceGroup.MEMBERS, ()):
member = SectorMemberRow.from_mapping(row)
if member.trade_date == target and member.stock_code in basics:
memberships.setdefault(member.sector_code, {})[member.stock_code] = (
member.stock_name
)
# Only the chosen sector's current-day facts are needed. Keep independent
# raw fields: normalized stock facts can suppress amounts when daily is missing.
sources = self.repository.load_publication_rows(
history.publication.publication_id,
(
PublicationSourceGroup.DAILY,
PublicationSourceGroup.MONEYFLOW_DC,
PublicationSourceGroup.MONEYFLOW,
),
trade_date=target,
ts_codes=tuple(sorted(memberships.get(sector_code, {}))),
)
daily = {
item.ts_code: item
for snapshot in snapshots.get(PublicationSourceGroup.DAILY, ())
for row in snapshot.rows
for row in sources.get(PublicationSourceGroup.DAILY, ())
for item in (DailyRow.from_mapping(row),)
if item.trade_date == target
}
main = {
item.ts_code: item
for snapshot in snapshots.get(PublicationSourceGroup.MONEYFLOW_DC, ())
for row in snapshot.rows
for row in sources.get(PublicationSourceGroup.MONEYFLOW_DC, ())
for item in (MoneyflowDcRow.from_mapping(row),)
if item.trade_date == target
}
active = {
item.ts_code: item
for snapshot in snapshots.get(PublicationSourceGroup.MONEYFLOW, ())
for row in snapshot.rows
for row in sources.get(PublicationSourceGroup.MONEYFLOW, ())
for item in (MoneyflowRow.from_mapping(row),)
if item.trade_date == target
}
@@ -414,6 +447,11 @@ def metric_at(
]
size = sum(row.rank_position is not None for row in pool)
row = next((row for row in pool if row.observation.sector_code == sector_code), None)
return _metric_from_ranking(row, size)
def _metric_from_ranking(row: RankedMetric | None, size: int) -> HistoryMetric:
"""Preserve missing metrics, stored percentiles and pool size independently."""
if row is None:
return HistoryMetric(pool_size=size)
percentile = row.rank_percentile
zhixing-server/src/zhixing_server/modules/sector_radar/domain/persistence.py
+40 -1
View File
@@ -12,13 +12,14 @@ from typing import Protocol
from .models import (
MembershipStatus,
MetricKind,
RadarPublication,
RankedMetric,
SectorDailyAggregate,
SectorType,
StockFactStatus,
)
from .source import SourceSnapshot
from .source import SourceScalar, SourceSnapshot
def _validate_digest(value: str, field_name: str) -> None:
@@ -151,6 +152,22 @@ class RankingRecord:
raise ValueError("publication_id must not be empty")
@dataclass(frozen=True, slots=True)
class HistoricalRanking:
"""A requested ranking plus its unfiltered publication/type/version pool size.
A pool without a requested sector still returns one entry with ranking=None,
so missing history retains the actual pool size instead of inventing zero.
"""
publication_id: str
sector_type: SectorType
metric_kind: MetricKind
metric_version: str
pool_size: int
ranking: RankedMetric | None
@dataclass(frozen=True, slots=True)
class DailyAggregateRecord:
"""One exact daily strategy input and optional source detail owned by a publication."""
@@ -230,6 +247,22 @@ class SectorRadarRepository(Protocol):
self, publication_id: str
) -> Sequence[PublicationSourceRecord]: ...
def load_publication_rows(
self,
publication_id: str,
source_groups: Sequence[PublicationSourceGroup],
*,
trade_date: date | None = None,
ts_codes: Sequence[str] | None = None,
) -> dict[PublicationSourceGroup, tuple[dict[str, SourceScalar], ...]]:
"""Read ordered source rows, optionally matching row trade_date and ts_code.
None means unfiltered; empty groups/codes return no rows. Projections do
not claim the source snapshot's row count or content hash. All reads
remain bound to the exact publication, including legacy publications.
"""
...
def mark_publication_sources_for_retry(
self,
publication_id: str,
@@ -307,6 +340,12 @@ class SectorRadarRepository(Protocol):
self, publication_ids: Sequence[str]
) -> Sequence[tuple[str, Sequence[RankedMetric]]]: ...
def load_ranked_history(
self, publication_ids: Sequence[str], sector_codes: Sequence[str]
) -> Sequence[HistoricalRanking]:
"""Read requested sectors and full pool counts from exact revisions."""
...
def load_daily_aggregate_history(
self, target_trade_date: date, *, limit_dates: int
) -> Sequence[SectorDailyAggregate]: ...
zhixing-server/src/zhixing_server/modules/sector_radar/infrastructure/memory.py
+58 -1
View File
@@ -8,6 +8,7 @@ from dataclasses import replace
from datetime import date, datetime
from ..domain.models import (
MetricKind,
PublicationStatus,
RadarPublication,
RankedMetric,
@@ -16,6 +17,7 @@ from ..domain.models import (
)
from ..domain.persistence import (
DailyAggregateRecord,
HistoricalRanking,
MembershipRecord,
PublicationSourceGroup,
PublicationSourceRecord,
@@ -25,7 +27,7 @@ from ..domain.persistence import (
StockMembershipEntry,
WriteCounts,
)
from ..domain.source import SourceSnapshot
from ..domain.source import SourceScalar, SourceSnapshot
class InMemorySectorRadarRepository:
@@ -98,6 +100,36 @@ class InMemorySectorRadarRepository:
)
)
def load_publication_rows(
self,
publication_id: str,
source_groups: Sequence[PublicationSourceGroup],
*,
trade_date: date | None = None,
ts_codes: Sequence[str] | None = None,
) -> dict[PublicationSourceGroup, tuple[dict[str, SourceScalar], ...]]:
"""Project exact source rows with the same filtering and order as PostgreSQL."""
groups = set(source_groups)
codes = None if ts_codes is None else set(ts_codes)
target = None if trade_date is None else trade_date.strftime("%Y%m%d")
result: dict[PublicationSourceGroup, list[dict[str, SourceScalar]]] = {}
for record in sorted(
self.publication_sources.values(),
key=lambda item: (item.source_group.value, item.source_order),
):
if record.publication_id != publication_id or record.source_group not in groups:
continue
for row in record.snapshot.rows:
if (
target is not None
and str(row.get("trade_date", "")).strip().replace("-", "") != target
):
continue
if codes is not None and str(row.get("ts_code", "")).strip() not in codes:
continue
result.setdefault(record.source_group, []).append(dict(row))
return {group: tuple(rows) for group, rows in result.items()}
def mark_publication_sources_for_retry(
self,
publication_id: str,
@@ -469,6 +501,31 @@ class InMemorySectorRadarRepository:
"""Read exact immutable revisions selected by the history reader."""
return tuple((key, self.load_rankings(key)) for key in publication_ids)
def load_ranked_history(
self, publication_ids: Sequence[str], sector_codes: Sequence[str]
) -> Sequence[HistoricalRanking]:
"""Keep unfiltered pool counts while returning only requested sector rows."""
if not publication_ids or not sector_codes:
return ()
wanted = set(sector_codes)
pools: dict[tuple[str, SectorType, MetricKind, str], list[RankedMetric]] = {}
for publication_id in publication_ids:
for row in self.load_rankings(publication_id):
observation = row.observation
key = (
publication_id,
observation.sector_type,
observation.metric_kind,
observation.metric_version,
)
pools.setdefault(key, []).append(row)
result: list[HistoricalRanking] = []
for key, rows in sorted(pools.items()):
size = sum(row.rank_position is not None for row in rows)
selected = [row for row in rows if row.observation.sector_code in wanted]
result.extend(HistoricalRanking(*key, size, row) for row in selected or [None])
return tuple(result)
def load_daily_aggregate_history(
self, target_trade_date: date, *, limit_dates: int
) -> Sequence[SectorDailyAggregate]:
zhixing-server/src/zhixing_server/modules/sector_radar/infrastructure/postgres.py
+94 -1
View File
@@ -26,6 +26,7 @@ from ..domain.models import (
)
from ..domain.persistence import (
DailyAggregateRecord,
HistoricalRanking,
MembershipRecord,
PublicationSourceGroup,
PublicationSourceRecord,
@@ -35,7 +36,7 @@ from ..domain.persistence import (
StockMembershipEntry,
WriteCounts,
)
from ..domain.source import SourceSnapshot
from ..domain.source import SourceScalar, SourceSnapshot
class SectorRadarRepositoryError(RuntimeError):
@@ -223,6 +224,46 @@ class PostgresSectorRadarRepository:
for row in rows
)
def load_publication_rows(
self,
publication_id: str,
source_groups: Sequence[PublicationSourceGroup],
*,
trade_date: date | None = None,
ts_codes: Sequence[str] | None = None,
) -> dict[PublicationSourceGroup, tuple[dict[str, SourceScalar], ...]]:
"""Filter immutable source payloads before transport and Python decoding.
Keep original source/row order so duplicate-key resolution is unchanged.
This is a row projection, not a snapshot with a misleading content hash.
Database failures are translated by the repository connection boundary.
"""
if not source_groups or ts_codes is not None and not ts_codes:
return {}
predicates = ["link.publication_id = %s", "link.source_group = ANY(%s)"]
parameters: list[object] = [publication_id, [group.value for group in source_groups]]
if trade_date is not None:
predicates.append("replace(btrim(item.value ->> 'trade_date'), '-', '') = %s")
parameters.append(trade_date.strftime("%Y%m%d"))
if ts_codes is not None:
predicates.append("btrim(item.value ->> 'ts_code') = ANY(%s)")
parameters.append(list(ts_codes))
query = (
"""
SELECT link.source_group,
jsonb_agg(item.value ORDER BY link.source_order, item.ordinality)
FROM sector_radar_publication_source AS link
JOIN sector_radar_source_snapshot AS snapshot ON snapshot.id = link.source_snapshot_id
CROSS JOIN LATERAL jsonb_array_elements(snapshot.payload)
WITH ORDINALITY AS item(value, ordinality)
WHERE """
+ " AND ".join(predicates)
+ " GROUP BY link.source_group"
)
with self._connection() as connection:
rows = connection.execute(query, tuple(parameters)).fetchall()
return {PublicationSourceGroup(row[0]): tuple(row[1]) for row in rows}
def mark_publication_sources_for_retry(
self,
publication_id: str,
@@ -870,6 +911,58 @@ class PostgresSectorRadarRepository:
grouped.setdefault(str(row[0]), []).append(self._ranking_from_row(row[1:]))
return tuple((key, tuple(grouped.get(key, ()))) for key in publication_ids)
def load_ranked_history(
self, publication_ids: Sequence[str], sector_codes: Sequence[str]
) -> Sequence[HistoricalRanking]:
"""Return requested rankings and complete pool sizes from immutable revisions.
Aggregate before filtering sectors: absent sectors must retain nonzero
pool sizes, and unavailable ranks must not count toward the pool.
"""
if not publication_ids or not sector_codes:
return ()
with self._connection() as connection:
rows = connection.execute(
"""
WITH pools AS (
SELECT publication_id, sector_type, metric_kind, metric_version,
COUNT(rank_position) AS pool_size
FROM sector_radar_ranking
WHERE publication_id = ANY(%s)
GROUP BY publication_id, sector_type, metric_kind, metric_version
)
SELECT pool.publication_id, pool.sector_type, pool.metric_kind,
pool.metric_version, pool.pool_size,
ranking.trade_date, ranking.sector_type, ranking.sector_code,
ranking.sector_name, ranking.metric_kind, ranking.metric_version,
ranking.implementation_kind, ranking.unit, ranking.metric_value,
ranking.quality, ranking.member_count, ranking.valid_sample_count,
ranking.membership_coverage, ranking.moneyflow_coverage,
ranking.rank_position, ranking.rank_percentile, ranking.rank_changes
FROM pools AS pool
LEFT JOIN sector_radar_ranking AS ranking
ON ranking.publication_id = pool.publication_id
AND ranking.sector_type = pool.sector_type
AND ranking.metric_kind = pool.metric_kind
AND ranking.metric_version = pool.metric_version
AND ranking.sector_code = ANY(%s)
ORDER BY pool.publication_id, pool.sector_type, pool.metric_kind,
ranking.rank_position NULLS LAST, ranking.sector_code
""",
(list(publication_ids), list(sector_codes)),
).fetchall()
return tuple(
HistoricalRanking(
str(row[0]),
SectorType(row[1]),
MetricKind(row[2]),
str(row[3]),
int(row[4]),
None if row[5] is None else self._ranking_from_row(row[5:]),
)
for row in rows
)
def load_daily_aggregate_history(
self, target_trade_date: date, *, limit_dates: int
) -> Sequence[SectorDailyAggregate]: