Wave I: 대량·다양 데이터 (다년치 135만 매입 → 총 1,130만 행)
기존 3개월치(~20만행)를 5.5년치 대량 히스토리로 확대. 성장추세·계절성· 가맹점 생애주기 반영. ID 오프셋 대역(1천만+)으로 기존/라이브 채번과 비충돌, 전부 idempotent 가드. - 98-bulk-seed.sql: 가맹점 40→200(MCC·요율·개점일·CLOSED/HOLD 다양) + 매입 135만건(2021-01~2026-04, 성장추세 연 0.5→1.4배·12월 계절피크·주중주말· 다양 채널/발급사/카드종류/금액구간). 27초. - 99-bulk-derive.sql: 승인 135만·원장 2.6M(RECON+SETTLE)·정산 1.29M· 가맹점집계 358k. 시퀀스 정렬. 31초. - 99b-bulk-econ.sql: 정산경제 파생 — 수수료3층분해 1.29M·프레젠트먼트 1.29M· ISO8583 클리어링 1.29M·순액정산(발급사×일) 16k·펀딩 355k. - 99c-bulk-index.sql: 운영 인덱스 14개(biz_date/merchant/status 등) + ANALYZE. 검증 (실측) - 총 1,130만 행, DB 1.8GB. 연도별 성장 2021년 14.5만→2025년 35.3만건. - 3층 항등식 1,293,145/1,293,145, 펀딩 항등식 355,284/355,284 (100% 성립). - 대시보드/정산경제 화면 정상: 활성가맹점 191, 총매입 3,997억, 입금률 95.7%. - 집계쿼리 0.5초(인덱스). 최근 3개월 라이브 데이터 무손상. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
hyeongwoo
parent
e1729d6edc
commit
ab7983318d
5 changed files with 245 additions and 1 deletions
102
db/schema.d/98-bulk-seed.sql
Normal file
102
db/schema.d/98-bulk-seed.sql
Normal file
|
|
@ -0,0 +1,102 @@
|
|||
-- ===========================================================================
|
||||
-- 98-bulk-seed.sql : 대량·다양 히스토리 시드 (다년치)
|
||||
-- 기존 90-seed(2026-04-20~07-19, 3개월)의 앞쪽에 2021-01-01~2026-04-19 히스토리를
|
||||
-- 붙여 볼륨을 수백만 건으로 확대한다. 성장추세·계절성·가맹점 생애주기 반영.
|
||||
-- ID 는 1천만 오프셋 대역 사용(기존 1000~9999 및 라이브 채번과 비충돌).
|
||||
-- idempotent: 이미 대량시드가 있으면(=오프셋 대역 존재) 건너뛴다.
|
||||
-- ===========================================================================
|
||||
SELECT setseed(0.73);
|
||||
|
||||
DO $$
|
||||
DECLARE v_exists bigint;
|
||||
BEGIN
|
||||
SELECT count(*) INTO v_exists FROM purchase WHERE purchase_id >= 10000000;
|
||||
IF v_exists > 0 THEN
|
||||
RAISE NOTICE '대량시드 이미 적재됨(% 건) — 건너뜀', v_exists;
|
||||
RETURN;
|
||||
END IF;
|
||||
|
||||
-- -------------------------------------------------------------------------
|
||||
-- 1. 가맹점 대량 확대 (M0041~M0200, 160개). MCC·업종·요율·개점일 다양.
|
||||
-- -------------------------------------------------------------------------
|
||||
INSERT INTO merchant (merchant_id, name, mdr_bps, daily_limit, status, created_at)
|
||||
SELECT 'M' || lpad(g::text, 4, '0'),
|
||||
(ARRAY['라온','블루','그린','썬','스카이','한빛','미소','골든','포레','오션',
|
||||
'스타','로얄','네오','에버','휘게','코지','퓨어','모던','정성','신선'])[1 + (g % 20)]
|
||||
|| ' ' ||
|
||||
(ARRAY['마트','식당','카페','약국','의원','주유소','서점','전자','헤어살롱','헬스장',
|
||||
'여행사','호텔','베이커리','문구','꽃집','세탁','안경','노래방','PC방','편의점'])[1 + ((g*7) % 20)]
|
||||
|| ' ' || (g - 40) || '호점',
|
||||
(ARRAY[15,18,20,22,24,25,26,27,28,29,30])[1 + (g % 11)],
|
||||
(ARRAY[10000000,30000000,50000000,100000000,200000000,300000000,500000000])[1 + (g % 7)],
|
||||
CASE WHEN g % 40 = 0 THEN 'CLOSED' WHEN g % 55 = 0 THEN 'HOLD' ELSE 'ACTIVE' END,
|
||||
timestamptz '2020-06-01' + ((g - 40) * interval '11 days')
|
||||
FROM generate_series(41, 200) g
|
||||
ON CONFLICT (merchant_id) DO NOTHING;
|
||||
|
||||
-- 신규 가맹점 MCC 매핑 (기존 mm_merch_mcc 확장)
|
||||
INSERT INTO mm_merch_mcc (merchant_id, mcc)
|
||||
SELECT 'M' || lpad(g::text, 4, '0'),
|
||||
(ARRAY['5411','5812','5814','5912','8062','5541','5942','5732','7230','7997',
|
||||
'4722','7011','5462','5943','5992','7216','8043','7929','7994','5411'])[1 + ((g*7) % 20)]
|
||||
FROM generate_series(41, 200) g
|
||||
ON CONFLICT (merchant_id) DO NOTHING;
|
||||
END $$;
|
||||
|
||||
-- ---------------------------------------------------------------------------
|
||||
-- 2. 대량 히스토리 매입 (2021-01-01 ~ 2026-04-19).
|
||||
-- 일 거래량 = 성장추세(연도↑) × 계절성(12월 피크) × 주중/주말.
|
||||
-- ---------------------------------------------------------------------------
|
||||
INSERT INTO purchase (purchase_id, merchant_id, amount, fee, net, status, biz_date,
|
||||
channel, issuer, category, created_at, updated_at)
|
||||
WITH days AS (
|
||||
SELECT d::date AS biz_date,
|
||||
extract(year FROM d)::int AS yy,
|
||||
extract(month FROM d)::int AS mm,
|
||||
extract(isodow FROM d)::int AS dow
|
||||
FROM generate_series('2021-01-01'::date, '2026-04-19'::date, '1 day') d
|
||||
),
|
||||
vol AS (
|
||||
SELECT biz_date, yy, mm, dow,
|
||||
-- 연 성장(2021=0.5 → 2026=1.4) × 계절(12월 1.6, 1·2월 0.8) × 주말감소
|
||||
GREATEST(50, (
|
||||
(0.5 + (yy - 2021) * 0.18)
|
||||
* CASE WHEN mm = 12 THEN 1.6 WHEN mm IN (1,2) THEN 0.8 WHEN mm IN (7,8) THEN 1.15 ELSE 1.0 END
|
||||
* CASE WHEN dow >= 6 THEN 0.45 ELSE 1.0 END
|
||||
* 900
|
||||
)::int) AS ntx
|
||||
FROM days
|
||||
),
|
||||
raw AS (
|
||||
SELECT 10000000 + row_number() OVER (ORDER BY v.biz_date, gs) AS purchase_id,
|
||||
v.biz_date,
|
||||
'M' || lpad((1 + floor(random()*199))::int::text, 4, '0') AS merchant_id,
|
||||
random() AS r_amt, random() AS r_ch, random() AS r_is, random() AS r_ct
|
||||
FROM vol v CROSS JOIN LATERAL generate_series(1, v.ntx) gs
|
||||
)
|
||||
SELECT r.purchase_id, r.merchant_id,
|
||||
(CASE WHEN r.r_amt < 0.55 THEN 3000 + floor(random()*47000)
|
||||
WHEN r.r_amt < 0.87 THEN 50000 + floor(random()*250000)
|
||||
WHEN r.r_amt < 0.97 THEN 300000+ floor(random()*1500000)
|
||||
ELSE 2000000 + floor(random()*5000000) END)::bigint / 100 * 100 AS amount,
|
||||
0, 0,
|
||||
-- 과거일수록 정산완료(S) 비중↑, 소량 취소(C)/무효(V)
|
||||
CASE WHEN r.r_ct < 0.955 THEN 'S' WHEN r.r_ct < 0.985 THEN 'C' ELSE 'V' END AS status,
|
||||
r.biz_date,
|
||||
CASE WHEN r.r_ch < 0.55 THEN 'POS' WHEN r.r_ch < 0.80 THEN 'EDC'
|
||||
WHEN r.r_ch < 0.93 THEN 'EDI' ELSE 'FOREIGN' END AS channel,
|
||||
CASE WHEN r.r_is < 0.26 THEN 'BC' WHEN r.r_is < 0.44 THEN 'KB'
|
||||
WHEN r.r_is < 0.60 THEN 'SHINHAN' WHEN r.r_is < 0.73 THEN 'HYUNDAI'
|
||||
WHEN r.r_is < 0.84 THEN 'SAMSUNG' WHEN r.r_is < 0.92 THEN 'LOTTE'
|
||||
WHEN r.r_is < 0.97 THEN 'NH' ELSE 'HANA' END AS issuer,
|
||||
CASE WHEN r.r_ch >= 0.93 THEN 'FX' ELSE 'GEN' END AS category,
|
||||
r.biz_date::timestamptz + interval '9 hours' + (random() * interval '12 hours'),
|
||||
r.biz_date::timestamptz + interval '9 hours' + (random() * interval '13 hours')
|
||||
FROM raw r
|
||||
WHERE NOT EXISTS (SELECT 1 FROM purchase p WHERE p.purchase_id >= 10000000 LIMIT 1);
|
||||
|
||||
-- 수수료/정산액 채우기 (mdr_bps 가맹점별)
|
||||
UPDATE purchase p SET fee = p.amount * m.mdr_bps / 10000,
|
||||
net = p.amount - p.amount * m.mdr_bps / 10000
|
||||
FROM merchant m WHERE m.merchant_id = p.merchant_id
|
||||
AND p.purchase_id >= 10000000 AND p.fee = 0;
|
||||
43
db/schema.d/99-bulk-derive.sql
Normal file
43
db/schema.d/99-bulk-derive.sql
Normal file
|
|
@ -0,0 +1,43 @@
|
|||
-- ===========================================================================
|
||||
-- 99-bulk-derive.sql : 대량 히스토리 매입(98-bulk)의 파생 테이블 적재.
|
||||
-- 승인/원장/정산/가맹점집계. ID 는 오프셋 대역(비충돌). idempotent 가드.
|
||||
-- ===========================================================================
|
||||
|
||||
-- 승인 (매입 1:1, 99%)
|
||||
INSERT INTO approval (approval_id, purchase_id, approved, approved_at)
|
||||
SELECT purchase_id, purchase_id, (random() < 0.99),
|
||||
created_at
|
||||
FROM purchase p WHERE p.purchase_id >= 10000000
|
||||
AND NOT EXISTS (SELECT 1 FROM approval a WHERE a.approval_id = p.purchase_id);
|
||||
|
||||
-- 원장: 정산완료(S) 건 RECON + SETTLE 이중기표
|
||||
INSERT INTO ledger (ledger_id, purchase_id, entry_type, amount, biz_date, created_at)
|
||||
SELECT 100000000 + row_number() OVER (ORDER BY purchase_id), purchase_id, 'RECON', net, biz_date,
|
||||
biz_date::timestamptz + interval '20 hours'
|
||||
FROM purchase WHERE purchase_id >= 10000000 AND status = 'S'
|
||||
AND NOT EXISTS (SELECT 1 FROM ledger l WHERE l.ledger_id >= 100000000 LIMIT 1);
|
||||
INSERT INTO ledger (ledger_id, purchase_id, entry_type, amount, biz_date, created_at)
|
||||
SELECT 200000000 + row_number() OVER (ORDER BY purchase_id), purchase_id, 'SETTLE', net, biz_date,
|
||||
biz_date::timestamptz + interval '21 hours'
|
||||
FROM purchase WHERE purchase_id >= 10000000 AND status = 'S'
|
||||
AND NOT EXISTS (SELECT 1 FROM ledger l WHERE l.ledger_id >= 200000000 LIMIT 1);
|
||||
|
||||
-- 정산 (S건, T+1 지급)
|
||||
INSERT INTO settlement (settlement_id, purchase_id, merchant_id, net, biz_date, status, settled_at)
|
||||
SELECT 30000000 + row_number() OVER (ORDER BY purchase_id), purchase_id, merchant_id, net, biz_date,
|
||||
'SETTLED', biz_date::timestamptz + interval '1 day 10 hours'
|
||||
FROM purchase WHERE purchase_id >= 10000000 AND status = 'S'
|
||||
AND NOT EXISTS (SELECT 1 FROM settlement s WHERE s.settlement_id >= 30000000 LIMIT 1);
|
||||
|
||||
-- 가맹점 일집계
|
||||
INSERT INTO merchant_summary (merchant_id, biz_date, txn_count, gross_amount, fee_amount, net_amount)
|
||||
SELECT merchant_id, biz_date, count(*), sum(amount), sum(fee), sum(net)
|
||||
FROM purchase WHERE purchase_id >= 10000000
|
||||
GROUP BY merchant_id, biz_date
|
||||
ON CONFLICT (merchant_id, biz_date) DO NOTHING;
|
||||
|
||||
-- 시퀀스 정렬 (라이브 채번이 오프셋 대역 뒤로 가도록)
|
||||
SELECT setval('purchase_seq', (SELECT max(purchase_id) FROM purchase));
|
||||
SELECT setval('approval_seq', (SELECT max(approval_id) FROM approval));
|
||||
SELECT setval('ledger_seq', (SELECT max(ledger_id) FROM ledger));
|
||||
SELECT setval('settlement_seq', (SELECT max(settlement_id) FROM settlement));
|
||||
77
db/schema.d/99b-bulk-econ.sql
Normal file
77
db/schema.d/99b-bulk-econ.sql
Normal file
|
|
@ -0,0 +1,77 @@
|
|||
-- ===========================================================================
|
||||
-- 99b-bulk-econ.sql : 대량 히스토리 매입의 정산 경제 파생.
|
||||
-- 수수료 3층 분해(MCC interchange)·프레젠트먼트·순액정산·펀딩·ISO8583 클리어링.
|
||||
-- 기존 96/97 econ 로직과 동일 산식. idempotent 가드(오프셋 대역 존재 검사).
|
||||
-- ===========================================================================
|
||||
|
||||
-- 1. 수수료 3층 분해 (MCC interchange 우선, 채널 폴백, scheme=BC)
|
||||
INSERT INTO st_fee_component (purchase_id, amount, interchange, scheme_fee, markup, mdr_total, card_type, scheme)
|
||||
SELECT p.purchase_id, p.amount,
|
||||
ic.amt,
|
||||
p.amount * 11 / 10000,
|
||||
GREATEST(p.fee - ic.amt - p.amount*11/10000, p.amount*8/10000),
|
||||
ic.amt + p.amount*11/10000
|
||||
+ GREATEST(p.fee - ic.amt - p.amount*11/10000, p.amount*8/10000),
|
||||
t.ct, 'BC'
|
||||
FROM purchase p
|
||||
CROSS JOIN LATERAL (
|
||||
SELECT (CASE WHEN (p.purchase_id % 10) < 3 THEN 'CHECK' ELSE 'CREDIT' END) AS ct
|
||||
) t
|
||||
CROSS JOIN LATERAL (
|
||||
SELECT p.amount * COALESCE(
|
||||
(SELECT r.rate_bps FROM mm_merch_mcc mm
|
||||
JOIN st_interchange_rate r ON r.card_type = t.ct AND r.mcc = mm.mcc
|
||||
AND r.channel='DEFAULT' AND r.txn_type='SALE'
|
||||
WHERE mm.merchant_id = p.merchant_id LIMIT 1),
|
||||
130) / 10000 AS amt
|
||||
) ic
|
||||
WHERE p.purchase_id >= 10000000 AND p.status = 'S'
|
||||
AND NOT EXISTS (SELECT 1 FROM st_fee_component f WHERE f.purchase_id >= 10000000 LIMIT 1);
|
||||
|
||||
-- 2. 프레젠트먼트(클리어링) — amount − interchange
|
||||
INSERT INTO cl_presentment (presentment_id, purchase_id, scheme, msg_class, clearing_amt, present_date, cycle, status)
|
||||
SELECT nextval('cl_present_seq'), fc.purchase_id, 'BC', 'FIRST_PRESENT',
|
||||
fc.amount - fc.interchange, p.biz_date, 'T+1', 'CLEARED'
|
||||
FROM st_fee_component fc JOIN purchase p ON p.purchase_id = fc.purchase_id
|
||||
WHERE fc.purchase_id >= 10000000
|
||||
AND NOT EXISTS (SELECT 1 FROM cl_presentment c WHERE c.purchase_id >= 10000000 LIMIT 1);
|
||||
|
||||
-- 3. ISO 8583 클리어링 PRESENTMENT (이중/단일)
|
||||
INSERT INTO mg_clearing (clr_id, purchase_id, scheme, msg_class, mti, msg_type, amount, biz_date, status)
|
||||
SELECT nextval('mg_clearing_seq'), fc.purchase_id, 'BC', 'PRESENTMENT', '1240',
|
||||
CASE WHEN fc.card_type='CHECK' THEN 'SINGLE' ELSE 'DUAL' END,
|
||||
fc.amount - fc.interchange, p.biz_date, 'ACCEPTED'
|
||||
FROM st_fee_component fc JOIN purchase p ON p.purchase_id = fc.purchase_id
|
||||
WHERE fc.purchase_id >= 10000000
|
||||
AND NOT EXISTS (SELECT 1 FROM mg_clearing c WHERE c.purchase_id >= 10000000 LIMIT 1);
|
||||
|
||||
-- 4. 순액정산 (발급사 BIN × 영업일 집계)
|
||||
INSERT INTO st_net_settlement (net_id, scheme, member_bin, biz_date, txn_count,
|
||||
gross_amt, interchange_amt, scheme_fee_amt, net_amt, direction, status)
|
||||
SELECT nextval('st_netset_seq'), 'BC', mb.bin, p.biz_date, count(*),
|
||||
sum(fc.amount), sum(fc.interchange), sum(fc.scheme_fee),
|
||||
sum(fc.amount - fc.interchange - fc.scheme_fee), 'RECEIVE', 'SETTLED'
|
||||
FROM st_fee_component fc
|
||||
JOIN purchase p ON p.purchase_id = fc.purchase_id
|
||||
JOIN st_member_bank mb ON mb.bank_code = p.issuer
|
||||
WHERE fc.purchase_id >= 10000000
|
||||
GROUP BY mb.bin, p.biz_date
|
||||
ON CONFLICT (scheme, member_bin, biz_date) DO NOTHING;
|
||||
|
||||
-- 5. 펀딩 항등식 (가맹점 × 영업일)
|
||||
INSERT INTO st_merchant_funding (funding_id, merchant_id, biz_date, gross_sales, refunds,
|
||||
chargebacks, fees, reserve, deposit, funding_model, status)
|
||||
SELECT nextval('st_funding_seq'), s.merchant_id, s.biz_date,
|
||||
s.gross, s.refunds, 0, s.fees, 0,
|
||||
s.gross - s.refunds - s.fees, 'NET_DAILY', 'FUNDED'
|
||||
FROM (
|
||||
SELECT p.merchant_id, p.biz_date,
|
||||
sum(CASE WHEN p.status='S' THEN p.amount ELSE 0 END) gross,
|
||||
sum(CASE WHEN p.status='C' THEN p.amount ELSE 0 END) refunds,
|
||||
coalesce(sum(fc.mdr_total),0) fees
|
||||
FROM purchase p LEFT JOIN st_fee_component fc ON fc.purchase_id=p.purchase_id
|
||||
WHERE p.purchase_id >= 10000000
|
||||
GROUP BY p.merchant_id, p.biz_date
|
||||
) s
|
||||
WHERE s.gross > 0
|
||||
ON CONFLICT (merchant_id, biz_date) DO NOTHING;
|
||||
20
db/schema.d/99c-bulk-index.sql
Normal file
20
db/schema.d/99c-bulk-index.sql
Normal file
|
|
@ -0,0 +1,20 @@
|
|||
-- ===========================================================================
|
||||
-- 99c-bulk-index.sql : 대량 데이터 운영 인덱스. 화면/조회 쿼리 응답성 확보.
|
||||
-- (실운영 매입시스템의 상시 인덱스에 해당) 전부 IF NOT EXISTS (idempotent).
|
||||
-- ===========================================================================
|
||||
CREATE INDEX IF NOT EXISTS ix_purchase_bizdate ON purchase(biz_date);
|
||||
CREATE INDEX IF NOT EXISTS ix_purchase_merch ON purchase(merchant_id);
|
||||
CREATE INDEX IF NOT EXISTS ix_purchase_status ON purchase(status);
|
||||
CREATE INDEX IF NOT EXISTS ix_purchase_merch_day ON purchase(merchant_id, biz_date);
|
||||
CREATE INDEX IF NOT EXISTS ix_settlement_bizdate ON settlement(biz_date);
|
||||
CREATE INDEX IF NOT EXISTS ix_settlement_merch ON settlement(merchant_id);
|
||||
CREATE INDEX IF NOT EXISTS ix_ledger_bizdate ON ledger(biz_date);
|
||||
CREATE INDEX IF NOT EXISTS ix_ledger_pid ON ledger(purchase_id);
|
||||
CREATE INDEX IF NOT EXISTS ix_cmlog_bizdate ON cm_log(biz_date);
|
||||
CREATE INDEX IF NOT EXISTS ix_cmlog_level ON cm_log(log_level);
|
||||
CREATE INDEX IF NOT EXISTS ix_funding_merch ON st_merchant_funding(merchant_id);
|
||||
CREATE INDEX IF NOT EXISTS ix_netset_bizdate ON st_net_settlement(biz_date);
|
||||
CREATE INDEX IF NOT EXISTS ix_clearing_bizdate ON mg_clearing(biz_date);
|
||||
CREATE INDEX IF NOT EXISTS ix_mercsum_day ON merchant_summary(biz_date);
|
||||
ANALYZE purchase; ANALYZE settlement; ANALYZE ledger; ANALYZE st_fee_component;
|
||||
ANALYZE st_merchant_funding; ANALYZE mg_clearing;
|
||||
Loading…
Add table
Add a link
Reference in a new issue