lapuda
d206b25fcf
- Add ExternalJobSourceAliasRequest class for managing source alias requests. - Create ExternalUploadClient to interact with external Excel upload system. - Introduce ExternalUploadFileMetadata to represent metadata of uploaded files. - Define IExternalJobImportService interface for external job import operations. - Update JobDataTrendServiceImpl to support dynamic source code filtering. - Modify PublicJobFairMapper.xml to ensure proper query handling. - Add unit tests for ExternalJobExcelParser and ExternalJobFingerprint. - Create script for applying external job import migrations. - Implement SQL migration for external job import, including new tables and columns.
222 lines
11 KiB
PL/PgSQL
222 lines
11 KiB
PL/PgSQL
-- 外部 Excel 岗位全量快照同步。
|
||
-- 执行目标：HighGo/PostgreSQL 的 shz schema。
|
||
-- 本脚本为增量 migration；不删除现有 job、company、row_work 或监测数据。
|
||
|
||
BEGIN;
|
||
|
||
-- 串行化本 migration 的并发执行，避免两个发布进程同时计算 MAX(id)+1。
|
||
-- 锁仅持续到本事务提交；重复执行不会改变已存在的业务记录或 Quartz 状态。
|
||
SELECT pg_advisory_xact_lock(hashtext('shz.migration_external_job_import'));
|
||
|
||
ALTER TABLE "shz"."job"
|
||
ADD COLUMN IF NOT EXISTS "external_import_flag" CHAR(1) NOT NULL DEFAULT '0',
|
||
ADD COLUMN IF NOT EXISTS "external_source_code" VARCHAR(64),
|
||
ADD COLUMN IF NOT EXISTS "external_source_name" VARCHAR(200),
|
||
ADD COLUMN IF NOT EXISTS "job_fingerprint" VARCHAR(64),
|
||
ADD COLUMN IF NOT EXISTS "job_content_fingerprint" VARCHAR(64),
|
||
ADD COLUMN IF NOT EXISTS "external_batch_id" VARCHAR(64),
|
||
ADD COLUMN IF NOT EXISTS "external_first_seen_at" TIMESTAMP,
|
||
ADD COLUMN IF NOT EXISTS "external_last_seen_at" TIMESTAMP;
|
||
|
||
COMMENT ON COLUMN "shz"."job"."external_import_flag" IS '是否外部 Excel 导入岗位：0否，1是';
|
||
COMMENT ON COLUMN "shz"."job"."job_fingerprint" IS '外部岗位身份指纹：FP_V1 + 来源编码 + 标准化岗位链接';
|
||
COMMENT ON COLUMN "shz"."job"."job_content_fingerprint" IS '外部岗位内容指纹：用于同一身份岗位的原地更新';
|
||
|
||
CREATE TABLE IF NOT EXISTS "shz"."job_import_batch" (
|
||
"batch_id" VARCHAR(64) PRIMARY KEY,
|
||
"file_id" VARCHAR(64) NOT NULL,
|
||
"file_name" VARCHAR(500) NOT NULL,
|
||
"file_sha256" VARCHAR(64) NOT NULL,
|
||
"snapshot_scope" TEXT,
|
||
"fingerprint_version" VARCHAR(16) NOT NULL,
|
||
"status" VARCHAR(20) NOT NULL,
|
||
"total_rows" INTEGER NOT NULL DEFAULT 0,
|
||
"valid_rows" INTEGER NOT NULL DEFAULT 0,
|
||
"invalid_rows" INTEGER NOT NULL DEFAULT 0,
|
||
"duplicate_rows" INTEGER NOT NULL DEFAULT 0,
|
||
"insert_count" INTEGER NOT NULL DEFAULT 0,
|
||
"update_count" INTEGER NOT NULL DEFAULT 0,
|
||
"keep_count" INTEGER NOT NULL DEFAULT 0,
|
||
"restore_count" INTEGER NOT NULL DEFAULT 0,
|
||
"delete_count" INTEGER NOT NULL DEFAULT 0,
|
||
"error_summary" TEXT,
|
||
"created_at" TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||
"updated_at" TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||
"completed_at" TIMESTAMP
|
||
);
|
||
|
||
CREATE TABLE IF NOT EXISTS "shz"."job_import_row" (
|
||
"row_id" BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY,
|
||
"batch_id" VARCHAR(64) NOT NULL,
|
||
"excel_row_number" INTEGER NOT NULL,
|
||
"source_name" VARCHAR(200),
|
||
"source_code" VARCHAR(64),
|
||
"job_fingerprint" VARCHAR(64),
|
||
"content_fingerprint" VARCHAR(64),
|
||
"validation_status" VARCHAR(20) NOT NULL,
|
||
"result_status" VARCHAR(20) NOT NULL,
|
||
"error_message" TEXT,
|
||
"raw_payload" TEXT,
|
||
"job_id" BIGINT,
|
||
"created_at" TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP
|
||
);
|
||
|
||
CREATE TABLE IF NOT EXISTS "shz"."job_import_source_alias" (
|
||
"alias_name" VARCHAR(200) PRIMARY KEY,
|
||
"source_code" VARCHAR(64) NOT NULL,
|
||
"website_id" BIGINT,
|
||
"is_active" CHAR(1) NOT NULL DEFAULT '1',
|
||
"snapshot_required" CHAR(1) NOT NULL DEFAULT '1',
|
||
"remark" VARCHAR(500),
|
||
"create_time" TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||
"update_time" TIMESTAMP
|
||
);
|
||
|
||
CREATE UNIQUE INDEX IF NOT EXISTS "uk_job_external_active_fingerprint"
|
||
ON "shz"."job" ("job_fingerprint")
|
||
WHERE "external_import_flag" = '1' AND "del_flag" = '0';
|
||
|
||
CREATE INDEX IF NOT EXISTS "idx_job_import_batch_sha_status"
|
||
ON "shz"."job_import_batch" ("file_sha256", "status");
|
||
CREATE INDEX IF NOT EXISTS "idx_job_import_row_batch"
|
||
ON "shz"."job_import_row" ("batch_id", "excel_row_number");
|
||
CREATE INDEX IF NOT EXISTS "idx_job_external_source_active"
|
||
ON "shz"."job" ("external_source_code", "external_import_flag", "del_flag");
|
||
|
||
-- Quartz 任务默认暂停，防止尚未配置 external-job-import URL/token 时发生无效拉取。
|
||
-- 完成环境配置后，在“定时任务”中启用即可；计划为每周三、周六 03:00。
|
||
INSERT INTO "shz"."sys_job" (
|
||
"job_id", "job_name", "job_group", "invoke_target", "cron_expression",
|
||
"misfire_policy", "concurrent", "status", "create_by", "create_time",
|
||
"update_by", "update_time", "remark"
|
||
)
|
||
SELECT
|
||
(SELECT COALESCE(MAX("job_id"), 0) + 1 FROM "shz"."sys_job"),
|
||
'外部岗位全量同步',
|
||
'DEFAULT',
|
||
'jobCron.importLatestExternalSnapshot()',
|
||
'0 0 3 ? * WED,SAT',
|
||
'3',
|
||
'1',
|
||
'1',
|
||
'system',
|
||
CURRENT_TIMESTAMP,
|
||
'',
|
||
NULL,
|
||
'外部 Excel 全量快照同步；配置 external-job-import 后由管理员启用'
|
||
WHERE NOT EXISTS (
|
||
SELECT 1 FROM "shz"."sys_job" WHERE "invoke_target" = 'jobCron.importLatestExternalSnapshot()'
|
||
);
|
||
|
||
-- 为样例及当前采集范围初始化来源管理记录。名称相近的来源不自动合并。
|
||
INSERT INTO "shz"."website_management" ("website_name", "website_url", "is_active", "data_type", "del_flag", "create_by", "create_time", "remark")
|
||
SELECT seed.website_name, '', '1', '2', '0', 'system', CURRENT_TIMESTAMP, '外部 Excel 岗位采集来源'
|
||
FROM (VALUES
|
||
('石河子招聘网'), ('智联招聘'), ('天互招聘网'), ('猎聘网'), ('乌鲁木齐快聘网'), ('前程无忧')
|
||
) AS seed(website_name)
|
||
WHERE NOT EXISTS (
|
||
SELECT 1 FROM "shz"."website_management" existing
|
||
WHERE existing."website_name" = seed.website_name AND existing."del_flag" = '0'
|
||
);
|
||
|
||
-- 以 alias_name 为幂等键；若此前仅完成了部分迁移，重跑时会补齐网站关联。
|
||
INSERT INTO "shz"."job_import_source_alias" AS target ("alias_name", "source_code", "website_id", "is_active", "snapshot_required", "remark")
|
||
SELECT seed.alias_name, seed.source_code, website."website_id", '1', '1', '外部 Excel ORG 来源映射'
|
||
FROM (VALUES
|
||
('石河子招聘网', 'shz_job_net'),
|
||
('智联招聘', 'zhaopin'),
|
||
('天互招聘网', 'tianhu_job'),
|
||
('猎聘网', 'liepin'),
|
||
('乌鲁木齐快聘网', 'wlmq_fast_job'),
|
||
('前程无忧', '51job')
|
||
) AS seed(alias_name, source_code)
|
||
LEFT JOIN "shz"."website_management" website ON website."website_name" = seed.alias_name AND website."del_flag" = '0'
|
||
ON CONFLICT ("alias_name") DO UPDATE
|
||
SET "source_code" = EXCLUDED."source_code",
|
||
"website_id" = EXCLUDED."website_id",
|
||
"is_active" = EXCLUDED."is_active",
|
||
"snapshot_required" = EXCLUDED."snapshot_required",
|
||
"remark" = EXCLUDED."remark",
|
||
"update_time" = CURRENT_TIMESTAMP
|
||
WHERE target."source_code" IS DISTINCT FROM EXCLUDED."source_code"
|
||
OR target."website_id" IS DISTINCT FROM EXCLUDED."website_id"
|
||
OR target."is_active" IS DISTINCT FROM EXCLUDED."is_active"
|
||
OR target."snapshot_required" IS DISTINCT FROM EXCLUDED."snapshot_required"
|
||
OR target."remark" IS DISTINCT FROM EXCLUDED."remark";
|
||
|
||
-- 四字菜单：来源管理、入库监测、采集趋势、统计分析。
|
||
INSERT INTO "shz"."sys_menu" (
|
||
"menu_id", "menu_name", "parent_id", "order_num", "path", "component", "query", "route_name",
|
||
"is_frame", "is_cache", "menu_type", "visible", "status", "perms", "icon", "create_by", "create_time", "remark"
|
||
)
|
||
SELECT (SELECT COALESCE(MAX("menu_id"), 0) + 1 FROM "shz"."sys_menu"),
|
||
'招聘信息采集监测分析', 0, 99, 'recruit-monitor', '', '', 'RecruitMonitor',
|
||
1, 0, 'M', '0', '0', 'cms:recruit:root', 'chart', 'system', CURRENT_TIMESTAMP, '外部岗位采集监测分析根菜单'
|
||
WHERE NOT EXISTS (SELECT 1 FROM "shz"."sys_menu" WHERE "perms" = 'cms:recruit:root');
|
||
|
||
INSERT INTO "shz"."sys_menu" (
|
||
"menu_id", "menu_name", "parent_id", "order_num", "path", "component", "query", "route_name",
|
||
"is_frame", "is_cache", "menu_type", "visible", "status", "perms", "icon", "create_by", "create_time", "remark"
|
||
)
|
||
SELECT ids."base_id" + ROW_NUMBER() OVER (ORDER BY seed.order_num),
|
||
seed.menu_name, parent."menu_id", seed.order_num, seed.path, seed.component, '', seed.route_name,
|
||
1, 0, 'C', '0', '0', seed.perms, 'table', 'system', CURRENT_TIMESTAMP, seed.remark
|
||
FROM (VALUES
|
||
('来源管理', 1, 'recruit/source', 'Recruit/Source/index', 'RecruitSource', 'cms:recruit:source:list', '外部岗位来源与别名管理'),
|
||
('入库监测', 2, 'recruit/monitor', 'Recruit/Monitor/index', 'RecruitMonitor', 'cms:recruit:monitor:list', '外部岗位导入批次监测'),
|
||
('采集趋势', 3, 'recruit/trend', 'Recruit/Trend/index', 'RecruitTrend', 'cms:recruit:trend:list', '外部岗位采集趋势'),
|
||
('统计分析', 4, 'recruit/analysis', 'Recruit/Analysis/index', 'RecruitAnalysis', 'cms:recruit:analysis:list', '外部岗位统计分析')
|
||
) AS seed(menu_name, order_num, path, component, route_name, perms, remark)
|
||
JOIN "shz"."sys_menu" parent ON parent."perms" = 'cms:recruit:root'
|
||
CROSS JOIN (SELECT COALESCE(MAX("menu_id"), 0) AS "base_id" FROM "shz"."sys_menu") ids
|
||
WHERE NOT EXISTS (SELECT 1 FROM "shz"."sys_menu" existing WHERE existing."perms" = seed.perms);
|
||
|
||
INSERT INTO "shz"."sys_menu" (
|
||
"menu_id", "menu_name", "parent_id", "order_num", "path", "component", "query", "route_name",
|
||
"is_frame", "is_cache", "menu_type", "visible", "status", "perms", "icon", "create_by", "create_time", "remark"
|
||
)
|
||
SELECT (SELECT COALESCE(MAX("menu_id"), 0) + 1 FROM "shz"."sys_menu"),
|
||
'手动导入', parent."menu_id", 1, '', '', '', '', 1, 0, 'F', '0', '0',
|
||
'cms:recruit:import:manual', '#', 'system', CURRENT_TIMESTAMP, '手动拉取并导入最新外部 Excel'
|
||
FROM "shz"."sys_menu" parent
|
||
WHERE parent."perms" = 'cms:recruit:monitor:list'
|
||
AND NOT EXISTS (SELECT 1 FROM "shz"."sys_menu" WHERE "perms" = 'cms:recruit:import:manual');
|
||
|
||
INSERT INTO "shz"."sys_menu" (
|
||
"menu_id", "menu_name", "parent_id", "order_num", "path", "component", "query", "route_name",
|
||
"is_frame", "is_cache", "menu_type", "visible", "status", "perms", "icon", "create_by", "create_time", "remark"
|
||
)
|
||
SELECT (SELECT COALESCE(MAX("menu_id"), 0) + 1 FROM "shz"."sys_menu"),
|
||
'来源维护', parent."menu_id", 1, '', '', '', '', 1, 0, 'F', '0', '0',
|
||
'cms:recruit:source:edit', '#', 'system', CURRENT_TIMESTAMP, '维护外部来源和来源别名'
|
||
FROM "shz"."sys_menu" parent
|
||
WHERE parent."perms" = 'cms:recruit:source:list'
|
||
AND NOT EXISTS (SELECT 1 FROM "shz"."sys_menu" WHERE "perms" = 'cms:recruit:source:edit');
|
||
|
||
INSERT INTO "shz"."sys_menu" (
|
||
"menu_id", "menu_name", "parent_id", "order_num", "path", "component", "query", "route_name",
|
||
"is_frame", "is_cache", "menu_type", "visible", "status", "perms", "icon", "create_by", "create_time", "remark"
|
||
)
|
||
SELECT (SELECT COALESCE(MAX("menu_id"), 0) + 1 FROM "shz"."sys_menu"),
|
||
'监测维护', parent."menu_id", 2, '', '', '', '', 1, 0, 'F', '0', '0',
|
||
'cms:recruit:monitor:edit', '#', 'system', CURRENT_TIMESTAMP, '维护导入监测记录'
|
||
FROM "shz"."sys_menu" parent
|
||
WHERE parent."perms" = 'cms:recruit:monitor:list'
|
||
AND NOT EXISTS (SELECT 1 FROM "shz"."sys_menu" WHERE "perms" = 'cms:recruit:monitor:edit');
|
||
|
||
INSERT INTO "shz"."sys_role_menu" ("role_id", "menu_id")
|
||
SELECT role."role_id", menu."menu_id"
|
||
FROM "shz"."sys_role" role
|
||
JOIN "shz"."sys_menu" menu ON menu."perms" IN (
|
||
'cms:recruit:root', 'cms:recruit:source:list', 'cms:recruit:monitor:list',
|
||
'cms:recruit:trend:list', 'cms:recruit:analysis:list', 'cms:recruit:import:manual',
|
||
'cms:recruit:source:edit', 'cms:recruit:monitor:edit'
|
||
)
|
||
WHERE role."role_key" = 'admin'
|
||
AND NOT EXISTS (
|
||
SELECT 1 FROM "shz"."sys_role_menu" existing
|
||
WHERE existing."role_id" = role."role_id" AND existing."menu_id" = menu."menu_id"
|
||
);
|
||
|
||
COMMIT;
|