worldquant-alpha-system/frontend/src/pages/DatasetPage.tsx

import {
  CatalogTableToolbar,
  CatalogIconAction,
  useCatalogColumns,
} from "../components/CatalogTableToolbar";
import {
  IconRefresh,
  IconDownload,
  IconAIFilledLevel3,
  IconEyeOpened,
  IconPlus,
} from "@douyinfe/semi-icons";
import type { ColumnProps } from "@douyinfe/semi-ui-19/lib/es/table/interface";
import { WorkspaceTable as Table } from "../components/WorkspaceTable";
import { useEffect, useState } from "react";
import {
  Banner,
  Button,
  Input,
  Modal,
  Pagination,
  TextArea,
  Toast,
  Tag,
} from "@douyinfe/semi-ui-19";
import {
  api,
  patch,
  post,
  queryString,
  displayValue,
  formatTime,
} from "../api";
import type { Account, Job } from "../types";
import type { PageContext, UIAction } from "../ai/types";
import { ResearchSelect } from "../research/ResearchSelect";
import { FieldAvailability } from "../research/FieldAvailability";
import { ScopeControls } from "../preparations/ScopeControls";
import { FieldTable } from "../preparations/FieldTable";
import { AddFieldsDialog } from "../preparations/FieldDirectory";
import {
  defaultScope,
  scopeKey,
  scopeLabel,
  type FieldRecord,
  type Page,
  type Preparation,
} from "../preparations/types";
import "../preparations/style.css";
type Entry = {
  id: string;
  name: string | null;
  description: string | null;
  category: string | null;
  subcategory: string | null;
  field_count: number | null;
  complete_count: number | null;
  collection_version: string | null;
  value_score: number | null;
  synced_at: string;
  research?: { note: string; version: number };
};
type Result = Page<Entry> & {
  categories: Record<string, string[]>;
  synced_at: string | null;
};
export function DatasetPage({
  active,
  version,
  onTask,
  onModal,
  onChat,
  onContext,
  suspended,
  account,
}: {
  active: boolean;
  version: string;
  onTask: () => void;
  onModal: (v: boolean) => void;
  onChat: () => void;
  onContext: (c: PageContext) => void;
  suspended: boolean;
  account: Account | null;
  jobs: Job[];
  action: UIAction | null;
}) {
  const [scope, setScope] = useState(defaultScope),
    [q, setQ] = useState(""),
    [category, setCategory] = useState(""),
    [subcategory, setSubcategory] = useState("");
  const [filterDraft, setFilterDraft] = useState({
    q: "",
    category: "",
    subcategory: "",
  });
  const columnSettings = useCatalogColumns("catalog-dataset-columns", [
    { key: "dataset", label: "数据集", required: true },
    { key: "category", label: "分类" },
    { key: "field_count", label: "字段数" },
    { key: "value_score", label: "价值评分" },
    { key: "status", label: "同步状态" },
    { key: "actions", label: "操作", required: true },
  ]);
  const [sort, setSort] = useState("name"),
    [direction, setDirection] = useState("asc"),
    [page, setPage] = useState(1),
    [pageSize, setPageSize] = useState(25),
    [rev, setRev] = useState(0);
  const [data, setData] = useState<Result>({
    items: [],
    total: 0,
    limit: 25,
    offset: 0,
    categories: {},
    synced_at: null,
  });
  const [detail, setDetail] = useState<Entry | null>(null),
    [fields, setFields] = useState<Page<FieldRecord>>({
      items: [],
      total: 0,
      limit: 25,
      offset: 0,
    });
  const [fieldQ, setFieldQ] = useState(""),
    [fieldPage, setFieldPage] = useState(1),
    [selected, setSelected] = useState<Record<string, FieldRecord>>({});
  const [adding, setAdding] = useState(false),
    [note, setNote] = useState(""),
    [noteVersion, setNoteVersion] = useState(1),
    [fieldDetail, setFieldDetail] = useState<FieldRecord | null>(null);
  const [error, setError] = useState(""),
    [busy, setBusy] = useState(false),
    [loading, setLoading] = useState(false),
    [copied, setCopied] = useState<Preparation | null>(null);
  const key = scopeKey(scope),
    visible = active && !suspended && !!detail;
  useEffect(() => {
    onModal(visible);
    return () => onModal(false);
  }, [visible, onModal]);
  useEffect(() => {
    if (active)
      onContext({
        page: "datasets",
        catalog_scope: scope,
        dataset_id: detail?.id,
      });
  }, [active, key, detail?.id, onContext]);
  useEffect(() => {
    setPage(1);
    setDetail(null);
    setSelected({});
  }, [key]);
  useEffect(() => {
    if (!active) return;
    const c = new AbortController();
    setLoading(true);
    api<Result>(
      `/catalog/datasets?${queryString({ ...scope, q, category, subcategory, sort, direction, limit: pageSize, offset: (page - 1) * pageSize })}`,
      { signal: c.signal },
    )
      .then(setData)
      .catch((e) => {
        if (!c.signal.aborted) setError(e.message);
      })
      .finally(() => {
        if (!c.signal.aborted) setLoading(false);
      });
    return () => c.abort();
  }, [
    active,
    key,
    q,
    category,
    subcategory,
    sort,
    direction,
    page,
    pageSize,
    version,
    rev,
  ]);
  useEffect(() => {
    if (!detail || !active) return;
    const c = new AbortController();
    api<Page<FieldRecord>>(
      `/catalog/fields?${queryString({ ...scope, dataset_id: detail.id, q: fieldQ, offset: (fieldPage - 1) * 25 })}`,
      { signal: c.signal },
    )
      .then(setFields)
      .catch((e) => {
        if (!c.signal.aborted) setError(e.message);
      });
    return () => c.abort();
  }, [active, detail?.id, key, fieldQ, fieldPage, version, rev]);
  async function work(fn: () => Promise<void>) {
    setBusy(true);
    setError("");
    try {
      await fn();
    } catch (e) {
      setError((e as Error).message);
    } finally {
      setBusy(false);
    }
  }
  async function sync(id?: string) {
    await post("/catalog/sync-jobs", { scope, dataset_id: id ?? null });
    onTask();
  }
  async function view(row: Entry) {
    const d = await api<Entry>(
      `/catalog/datasets/${encodeURIComponent(row.id)}?${queryString(scope)}`,
    );
    setDetail(d);
    setFieldDetail(null);
    setNote(d.research?.note ?? "");
    setNoteVersion(d.research?.version ?? 1);
    setFieldQ("");
    setFieldPage(1);
    setSelected({});
  }
  async function use(row: Entry) {
    const copy = await post<Preparation>("/data-preparations/from-dataset", {
      scope,
      dataset_id: row.id,
      collection_version: row.collection_version,
    });
    setCopied(copy);
    Toast.success(`已复制到数据准备：${copy.name}`);
  }
  async function openField(f: FieldRecord) {
    const d = await api<Entry>(
      `/catalog/datasets/${encodeURIComponent(f.dataset_id)}/fields/${encodeURIComponent(f.id)}?${queryString(scope)}`,
    );
    setFieldDetail(f);
    setNote(d.research?.note ?? "");
    setNoteVersion(d.research?.version ?? 1);
  }
  async function saveNote() {
    const suffix = fieldDetail
      ? `/fields/${encodeURIComponent(fieldDetail.id)}`
      : "";
    const n = await patch<{ note: string; version: number }>(
      `/catalog/datasets/${encodeURIComponent(detail!.id)}${suffix}/research?${queryString(scope)}`,
      { note, version: noteVersion },
    );
    setNoteVersion(n.version);
    Toast.success("备注已保存");
  }
  const columns: ColumnProps<Entry>[] = [
    {
      title: "数据集",
      width: 280,
      render: (_, r) => (
        <Button
          theme="borderless"
          title={r.name || r.id}
          onClick={() => void work(() => view(r))}
        >
          {r.name || r.id}
        </Button>
      ),
    },
    { title: "分类", dataIndex: "category", render: displayValue },
    {
      title: "字段数",
      render: (_, r) => r.complete_count ?? displayValue(r.field_count),
    },
    { title: "价值评分", dataIndex: "value_score", render: displayValue },
    {
      title: "同步状态",
      render: (_, r) => (
        <Tag color={r.collection_version ? "green" : "grey"}>
          {r.collection_version ? "已同步字段" : "待同步字段"}
        </Tag>
      ),
    },
    {
      title: "操作",
      width: 140,
      render: (_, r) => (
        <div className="workspace-cell-actions">
          <CatalogIconAction
            label="查看"
            icon={<IconEyeOpened aria-hidden="true" />}
            disabled={busy}
            onClick={() => void work(() => view(r))}
          />
          <CatalogIconAction
            label="同步"
            icon={<IconDownload aria-hidden="true" />}
            disabled={busy}
            onClick={() => void work(() => sync(r.id))}
          />
          <CatalogIconAction
            label="使用"
            icon={<IconPlus aria-hidden="true" />}
            disabled={busy || !r.collection_version}
            title={!r.collection_version ? "请先同步全部字段" : "使用"}
            onClick={() => void work(() => use(r))}
          />
        </div>
      ),
    },
  ];
  const visibleColumns = columns.filter((_, index) =>
    columnSettings.visible.includes(columnSettings.choices[index].key),
  );
  return (
    <section className="preparation-page">
      {error && <Banner type="danger" description={error} />}
      {copied && (
        <Banner
          type="success"
          description={
            <>
              {copied.name} 已创建{" "}
              <a href={`#preparations?id=${copied.id}`}>打开数据准备</a>
            </>
          }
        />
      )}
      <CatalogTableToolbar
        scope={
          <ScopeControls
            inline
            value={scope}
            onChange={setScope}
            active={active}
            revision={version}
          />
        }
        label="数据目录"
        active={active && !suspended && !detail}
        filterCount={[q, category, subcategory].filter(Boolean).length}
        columns={columnSettings}
        filters={
          <div className="catalog-filter-grid">
            <label className="catalog-filter-wide">
              搜索
              <Input
                aria-label="搜索数据集"
                placeholder="搜索数据集名称或 ID"
                value={filterDraft.q}
                onChange={(q) => setFilterDraft({ ...filterDraft, q })}
                showClear
              />
            </label>
            <label>
              分类
              <ResearchSelect
                label="分类"
                placeholder="全部分类"
                showClear
                value={filterDraft.category || undefined}
                optionList={Object.keys(data.categories).map((v) => ({
                  value: v,
                  label: v,
                }))}
                onChange={(v) =>
                  setFilterDraft({
                    ...filterDraft,
                    category: String(v ?? ""),
                    subcategory: "",
                  })
                }
              />
            </label>
            <label>
              子分类
              <ResearchSelect
                label="子分类"
                placeholder="全部子分类"
                showClear
                value={filterDraft.subcategory || undefined}
                optionList={(data.categories[filterDraft.category] ?? []).map(
                  (v) => ({ value: v, label: v }),
                )}
                onChange={(v) =>
                  setFilterDraft({
                    ...filterDraft,
                    subcategory: String(v ?? ""),
                  })
                }
              />
            </label>
          </div>
        }
        onApply={() => {
          setQ(filterDraft.q);
          setCategory(filterDraft.category);
          setSubcategory(filterDraft.subcategory);
          setPage(1);
        }}
        onReset={() => {
          setFilterDraft({ q: "", category: "", subcategory: "" });
          setQ("");
          setCategory("");
          setSubcategory("");
          setPage(1);
        }}
        sort={{
          value: sort,
          direction,
          options: [
            { value: "name", label: "名称" },
            { value: "field_count", label: "字段数" },
            { value: "value_score", label: "价值评分" },
          ],
          onChange: (value, dir) => {
            setSort(value);
            setDirection(dir);
            setPage(1);
          },
        }}
        actions={
          <>
            <CatalogIconAction
              label="刷新"
              icon={<IconRefresh aria-hidden="true" />}
              onClick={() => setRev((v) => v + 1)}
            />
            <CatalogIconAction
              label="AI 助手"
              icon={<IconAIFilledLevel3 aria-hidden="true" />}
              onClick={onChat}
            />
            <Button
              theme="solid"
              icon={<IconDownload aria-hidden="true" />}
              disabled={busy}
              onClick={() => void work(() => sync())}
            >
              同步目录
            </Button>
          </>
        }
      />
      <Table<Entry>
        fill
        rowKey="id"
        size="small"
        dataSource={data.items}
        pagination={false}
        loading={loading}
        scroll={{
          x: visibleColumns.reduce((sum, c) => sum + Number(c.width ?? 140), 0),
          y: "100%",
        }}
        columns={visibleColumns}
      />
      <div className="table-pagination workspace-table-footer">
        <span>
          {data.total} 个数据集 ·{" "}
          {formatTime(data.synced_at, account?.timezone)}
        </span>
        <Pagination
          currentPage={page}
          total={data.total}
          pageSize={pageSize}
          showSizeChanger
          pageSizeOpts={[25, 50, 100]}
          onPageSizeChange={(size) => {
            setPageSize(size);
            setPage(1);
          }}
          onPageChange={setPage}
        />
      </div>
      <Modal
        visible={visible && !adding}
        title={fieldDetail?.id ?? detail?.name ?? "数据集详情"}
        width="min(1200px,96vw)"
        footer={null}
        closeOnEsc
        onCancel={() => {
          if (fieldDetail) void work(() => view(detail!));
          else setDetail(null);
        }}
      >
        {error && <Banner type="danger" description={error} />}
        <div className="preparation-detail">
          <p>
            {scopeLabel(scope)} · 数据集 {detail?.id}
          </p>
          <p>{displayValue(fieldDetail?.description ?? detail?.description)}</p>
          <div className="preparation-tools">
            <Button onClick={onChat}>AI 助手</Button>
            <Button disabled={busy} onClick={() => void work(saveNote)}>
              保存备注
            </Button>
            <Button
              onClick={() =>
                void work(() =>
                  fieldDetail ? openField(fieldDetail) : view(detail!),
                )
              }
            >
              载入最新记录
            </Button>
          </div>
          <TextArea
            aria-label="研究备注"
            value={note}
            onChange={setNote}
            maxCount={20000}
          />
          {fieldDetail ? (
            <FieldAvailability field={fieldDetail.id} scope={scope} />
          ) : (
            <>
              <div className="preparation-tools">
                <Input
                  aria-label="数据集内搜索字段"
                  placeholder="搜索字段"
                  value={fieldQ}
                  onChange={(v) => {
                    setFieldQ(v);
                    setFieldPage(1);
                  }}
                />
                <Button
                  disabled={!Object.keys(selected).length}
                  onClick={() => setAdding(true)}
                >
                  加入数据准备
                </Button>
                <Button
                  disabled={busy || !detail?.collection_version}
                  onClick={() => void work(() => use(detail!))}
                >
                  使用全部字段
                </Button>
              </div>
              <FieldTable
                items={fields.items}
                selected={Object.keys(selected)}
                onOpen={(f) => void work(() => openField(f))}
                onSelect={(ids) =>
                  setSelected((old) => {
                    const next: Record<string, FieldRecord> = {};
                    for (const k of ids) {
                      const f =
                        fields.items.find(
                          (x) => `${x.dataset_id}|${x.id}` === k,
                        ) ?? old[k];
                      if (f) next[k] = f;
                    }
                    return next;
                  })
                }
              />
              <Pagination
                currentPage={fieldPage}
                total={fields.total}
                pageSize={25}
                onPageChange={setFieldPage}
              />
            </>
          )}
        </div>
      </Modal>
      {adding && (
        <AddFieldsDialog
          fields={Object.values(selected)}
          onClose={() => setAdding(false)}
        />
      )}
    </section>
  );
}