feat(catalog): preserve and display dataset value scores
Deploy production / deploy (push) Successful in 51s
Deploy production / deploy (push) Successful in 51s
This commit is contained in:
@@ -37,7 +37,7 @@ class CatalogFilters(Scope):
|
|||||||
field_type: str | None = None
|
field_type: str | None = None
|
||||||
coverage_min: float | None = Field(default=None, ge=0, le=1)
|
coverage_min: float | None = Field(default=None, ge=0, le=1)
|
||||||
sort: Literal[
|
sort: Literal[
|
||||||
"id", "name", "category", "field_count", "coverage", "user_count", "alpha_count", "field_type"
|
"id", "name", "category", "field_count", "coverage", "user_count", "alpha_count", "field_type", "value_score"
|
||||||
] = "name"
|
] = "name"
|
||||||
direction: Literal["asc", "desc"] = "asc"
|
direction: Literal["asc", "desc"] = "asc"
|
||||||
limit: int = Field(default=25, ge=1, le=100)
|
limit: int = Field(default=25, ge=1, le=100)
|
||||||
@@ -80,6 +80,7 @@ class EntryOutput(BaseModel):
|
|||||||
category: str | None
|
category: str | None
|
||||||
subcategory: str | None
|
subcategory: str | None
|
||||||
field_type: str | None
|
field_type: str | None
|
||||||
|
value_score: float | None = None
|
||||||
coverage: float | None
|
coverage: float | None
|
||||||
user_count: int | None
|
user_count: int | None
|
||||||
alpha_count: int | None
|
alpha_count: int | None
|
||||||
|
|||||||
@@ -55,6 +55,7 @@ def normalize(raw, dataset_id):
|
|||||||
subcategory=label(raw.get("subcategory")),
|
subcategory=label(raw.get("subcategory")),
|
||||||
field_type=label(raw.get("type")) if dataset_id else None,
|
field_type=label(raw.get("type")) if dataset_id else None,
|
||||||
coverage=coverage,
|
coverage=coverage,
|
||||||
|
value_score=number(raw.get("valueScore")) if not dataset_id else None,
|
||||||
user_count=number(raw.get("userCount"), True),
|
user_count=number(raw.get("userCount"), True),
|
||||||
alpha_count=number(raw.get("alphaCount"), True),
|
alpha_count=number(raw.get("alphaCount"), True),
|
||||||
field_count=number(raw.get("fieldCount"), True),
|
field_count=number(raw.get("fieldCount"), True),
|
||||||
|
|||||||
@@ -357,6 +357,7 @@ class CatalogEntry(Base):
|
|||||||
subcategory: Mapped[str | None] = mapped_column(String(200))
|
subcategory: Mapped[str | None] = mapped_column(String(200))
|
||||||
field_type: Mapped[str | None] = mapped_column(String(100))
|
field_type: Mapped[str | None] = mapped_column(String(100))
|
||||||
coverage: Mapped[float | None] = mapped_column(Float)
|
coverage: Mapped[float | None] = mapped_column(Float)
|
||||||
|
value_score: Mapped[float | None] = mapped_column(Float)
|
||||||
user_count: Mapped[int | None] = mapped_column(Integer)
|
user_count: Mapped[int | None] = mapped_column(Integer)
|
||||||
alpha_count: Mapped[int | None] = mapped_column(Integer)
|
alpha_count: Mapped[int | None] = mapped_column(Integer)
|
||||||
field_count: Mapped[int | None] = mapped_column(Integer)
|
field_count: Mapped[int | None] = mapped_column(Integer)
|
||||||
|
|||||||
@@ -0,0 +1,17 @@
|
|||||||
|
"""Preserve the upstream dataset value score; old snapshots remain unknown."""
|
||||||
|
|
||||||
|
import sqlalchemy as sa
|
||||||
|
from alembic import op
|
||||||
|
|
||||||
|
revision = "0009"
|
||||||
|
down_revision = "0008"
|
||||||
|
branch_labels = None
|
||||||
|
depends_on = None
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade():
|
||||||
|
op.add_column("catalog_entries", sa.Column("value_score", sa.Float(), nullable=True))
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade():
|
||||||
|
op.drop_column("catalog_entries", "value_score")
|
||||||
@@ -37,6 +37,7 @@ def catalog_response(request, fields=None):
|
|||||||
"category": {"name": "基本面"},
|
"category": {"name": "基本面"},
|
||||||
"subcategory": {"name": "财务报表"},
|
"subcategory": {"name": "财务报表"},
|
||||||
"fieldCount": 123,
|
"fieldCount": 123,
|
||||||
|
"valueScore": 2.0,
|
||||||
"description": "合成数据,仅用于验收",
|
"description": "合成数据,仅用于验收",
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
@@ -45,6 +46,7 @@ def catalog_response(request, fields=None):
|
|||||||
"category": {"name": "新闻"},
|
"category": {"name": "新闻"},
|
||||||
"subcategory": {"name": "情绪"},
|
"subcategory": {"name": "情绪"},
|
||||||
"fieldCount": 3,
|
"fieldCount": 3,
|
||||||
|
"valueScore": 0.0,
|
||||||
},
|
},
|
||||||
{"id": "TEST_UNKNOWN", "name": "TEST 未分类", "fieldCount": 0},
|
{"id": "TEST_UNKNOWN", "name": "TEST 未分类", "fieldCount": 0},
|
||||||
]
|
]
|
||||||
|
|||||||
@@ -279,3 +279,17 @@ async def test_dynamic_platform_scopes_and_validation(catalog):
|
|||||||
runner.client.disconnect()
|
runner.client.disconnect()
|
||||||
assert (await client.get(BASE + "/scopes")).status_code == 409
|
assert (await client.get(BASE + "/scopes")).status_code == 409
|
||||||
assert (await search(client))["total"] == 0
|
assert (await search(client))["total"] == 0
|
||||||
|
|
||||||
|
|
||||||
|
async def test_dataset_value_score_survives_sync_and_sorts_null_last(catalog):
|
||||||
|
client, _, _ = catalog
|
||||||
|
assert (await sync(catalog))["status"] == "completed"
|
||||||
|
rows = (await search(client, sort="value_score", direction="desc"))["items"]
|
||||||
|
assert [(row["id"], row["value_score"]) for row in rows] == [
|
||||||
|
("TEST_FIN", 2.0), ("TEST_NEWS", 0.0), ("TEST_UNKNOWN", None)
|
||||||
|
]
|
||||||
|
rows = (await search(client, sort="value_score", direction="asc"))["items"]
|
||||||
|
assert [row["value_score"] for row in rows] == [0.0, 2.0, None]
|
||||||
|
assert (await search(client, "/datasets/TEST_FIN"))["value_score"] == 2.0
|
||||||
|
assert (await search(client, "/datasets/TEST_NEWS"))["value_score"] == 0.0
|
||||||
|
assert (await search(client, "/datasets/TEST_UNKNOWN"))["value_score"] is None
|
||||||
|
|||||||
@@ -66,6 +66,7 @@ type Entry = {
|
|||||||
subcategory: string | null;
|
subcategory: string | null;
|
||||||
field_type: string | null;
|
field_type: string | null;
|
||||||
coverage: number | null;
|
coverage: number | null;
|
||||||
|
value_score: number | null;
|
||||||
user_count: number | null;
|
user_count: number | null;
|
||||||
alpha_count: number | null;
|
alpha_count: number | null;
|
||||||
field_count: number | null;
|
field_count: number | null;
|
||||||
@@ -732,6 +733,12 @@ export function DatasetPage({
|
|||||||
width: 120,
|
width: 120,
|
||||||
render: (_, row) => row.complete_count ?? displayValue(row.field_count),
|
render: (_, row) => row.complete_count ?? displayValue(row.field_count),
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
title: sortTitle("价值评分", "value_score"),
|
||||||
|
width: 120,
|
||||||
|
render: (_, row) =>
|
||||||
|
row.value_score == null ? "未提供" : row.value_score.toFixed(1),
|
||||||
|
},
|
||||||
{
|
{
|
||||||
title: "同步状态",
|
title: "同步状态",
|
||||||
width: 120,
|
width: 120,
|
||||||
@@ -1057,7 +1064,7 @@ export function DatasetPage({
|
|||||||
rowKey="id"
|
rowKey="id"
|
||||||
size="small"
|
size="small"
|
||||||
pagination={false}
|
pagination={false}
|
||||||
scroll={{ x: 884, y: "100%" }}
|
scroll={{ x: 1004, y: "100%" }}
|
||||||
empty={
|
empty={
|
||||||
<Empty
|
<Empty
|
||||||
description={
|
description={
|
||||||
@@ -1211,6 +1218,10 @@ export function DatasetPage({
|
|||||||
: {
|
: {
|
||||||
分类: displayValue(detailData.category),
|
分类: displayValue(detailData.category),
|
||||||
子分类: displayValue(detailData.subcategory),
|
子分类: displayValue(detailData.subcategory),
|
||||||
|
"价值评分 (Value Score)":
|
||||||
|
detailData.value_score == null
|
||||||
|
? "未提供"
|
||||||
|
: detailData.value_score.toFixed(2),
|
||||||
字段数:
|
字段数:
|
||||||
detailData.complete_count ??
|
detailData.complete_count ??
|
||||||
displayValue(detailData.field_count),
|
displayValue(detailData.field_count),
|
||||||
|
|||||||
Reference in New Issue
Block a user