Skip to content

Commit

Permalink
Add feature desp
Browse files Browse the repository at this point in the history
  • Loading branch information
@auxten
auxten committed Dec 21, 2022
1 parent fb1e5bb commit b77092a
Show file tree
Hide file tree
Showing 3 changed files with 71 additions and 68 deletions.
2 changes: 2 additions & 0 deletions README.md
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
# go-sql-lineage
Pure golang SQL lineage analysis toolkit

1. SQL:2011 based on [postgresql-parser](https://github.com/auxten/postgresql-parser)
1. Very complex SQL tested, see [SQLs](test/sql/sqls.go), 65.7% test coverage

## Quick Start

Expand Down
6 changes: 3 additions & 3 deletions lineage/from_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@ func TestGetProvidedColumns(t *testing.T) {
},
},
sql.SQLNestedSubquery,
"[employees.job_id employees.salary p.id p.score maxavg.max(myavg) maxavg.job_id]",
"[employees.job_id:Physical employees.salary:Physical p.id:Physical p.score:Physical maxavg.max(myavg):Unknown maxavg.job_id:Unknown]",
"[employees p maxavg]",
},
{
Expand All @@ -53,7 +53,7 @@ func TestGetProvidedColumns(t *testing.T) {
},
},
sql.SQLUnion,
"[type contactname city country]",
"[type:Unknown contactname:Physical city:Physical country:Physical]",
"[]",
},
{
Expand Down Expand Up @@ -96,7 +96,7 @@ func TestGetProvidedColumns(t *testing.T) {
},
},
sql.SQLmj,
"[clerk.rowid clerk.oper_no clerk.oper_name clerk.cert_no clerk.oper_no cifo.rowid cifo.cert_no cifo.cust_no basic.cust_no basic.acct_no basic.unnecessary acct.sub_code acct.acct_no acct.acct_bal]",
"[clerk.rowid:Physical clerk.oper_no:Physical clerk.oper_name:Physical clerk.cert_no:Physical clerk.oper_no:Physical cifo.rowid:Physical cifo.cert_no:Physical cifo.cust_no:Physical basic.cust_no:Physical basic.acct_no:Physical basic.unnecessary:Physical acct.sub_code:Physical acct.acct_no:Physical acct.acct_bal:Physical]",
"[clerk cifo basic acct]",
},
}
Expand Down
131 changes: 66 additions & 65 deletions lineage/normalize_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -75,12 +75,12 @@ func TestNormalizeASTWithoutInputSchema(t *testing.T) {
{
sql.SQLMultiFrom,
sql.SQLMultiFromNormal,
[]string{"a", "b", "ct4", "e", "t31.c", "t31.d"},
[]string{"a:Unknown", "b:Unknown", "ct4:Unknown", "e:Unknown", "t31.c:Physical", "t31.d:Physical"},
},
{
sql.SQLmj,
sql.SQLmjNormal,
[]string{"comc_clerk.cert_no", "comc_clerk.oper_name", "comc_clerk.oper_no", "comr_cifbinfo.cert_no", "comr_cifbinfo.cust_no", "savb_acctinfo_chk.acct_bal", "savb_acctinfo_chk.acct_no", "savb_acctinfo_chk.sub_code", "savb_basicinfo.acct_no", "savb_basicinfo.cust_no", "savb_basicinfo.unnecessary"},
[]string{"comc_clerk.cert_no:Physical", "comc_clerk.oper_name:Physical", "comc_clerk.oper_no:Physical", "comr_cifbinfo.cert_no:Physical", "comr_cifbinfo.cust_no:Physical", "savb_acctinfo_chk.acct_bal:Physical", "savb_acctinfo_chk.acct_no:Physical", "savb_acctinfo_chk.sub_code:Physical", "savb_basicinfo.acct_no:Physical", "savb_basicinfo.cust_no:Physical", "savb_basicinfo.unnecessary:Physical"},
},
{
`select marr
Expand Down Expand Up @@ -114,77 +114,78 @@ ORDER BY
table1.sex DESC
LIMIT
1`,
[]string{"root_loan_mock_v4.age", "root_loan_mock_v4.label", "root_loan_mock_v4.marr_stat_cd", "table1.sex"},
[]string{"root_loan_mock_v4.age:Physical", "root_loan_mock_v4.label:Physical", "root_loan_mock_v4.marr_stat_cd:Physical", "table1.sex:Physical"},
},
{
sql.SQL246,
sql.SQL246Normal,
[]string{"acct_term_temp.agreement_id",
"acct_term_temp.term",
"acct_term_temp.term_unit_cd",
"agreement_cash_pool_temp.cash_pool_group",
"agreement_cash_pool_temp.tid",
"agreement_item_temp.agreement_category_cd",
"agreement_item_temp.agreement_id",
"agreement_item_temp.agreement_mdfr",
"agreement_item_temp.agreement_stat_cd",
"agreement_item_temp.category_cd",
"agreement_item_temp.ccy_cd",
"agreement_item_temp.close_dt",
"agreement_item_temp.cur_bal",
"agreement_item_temp.item_id",
"agreement_item_temp.mature_dt",
"agreement_item_temp.open_acct_amt",
"agreement_item_temp.party_id",
"agreement_item_temp.sign_dt",
"agreement_item_temp.sign_org",
"agreement_item_temp.src_sys",
"agreement_item_temp.st_int_dt",
"s04_zmq_acc_cur.customer",
"s04_zmq_acc_cur.tid",
"t03_acct.agreement_id",
"t03_acct.dep_exchg_ind",
"t03_acct.fcy_spec_acct_id_type",
"t03_acct.sleep_acct_ind",
"t03_agreement_agt_h.agreement_id",
"t03_agreement_agt_h.agt_open_acct_verify_situati",
"t03_agreement_agt_h.agter_ident_info_category_cd",
"t03_agreement_agt_h.agter_ident_info_content",
"t03_agreement_agt_h.agter_nationality_cd",
"t03_agreement_agt_h.agter_nm",
"t03_agreement_agt_h.agter_tel",
"t03_agreement_agt_h.end_dt",
"t03_agreement_agt_h.st_dt",
"t03_agreement_int_h.agreement_id",
"t03_agreement_int_h.agreement_mdfr",
"t03_agreement_int_h.end_dt",
"t03_agreement_int_h.int_type_cd",
"t03_agreement_int_h.intr",
"t03_agreement_int_h.st_dt",
"t03_agreement_medium_rela_h.agreement_id",
"t03_agreement_medium_rela_h.agreement_medium_rela_type_cd",
"t03_agreement_medium_rela_h.end_dt",
"t03_agreement_medium_rela_h.medium_id",
"t03_agreement_medium_rela_h.st_dt",
"t03_agreement_pty_rela_h_temp.agreement_id",
"t03_agreement_pty_rela_h_temp.party_id",
"t03_agreement_rela_h_temp.agreement_id",
"t03_agreement_rela_h_temp.assoc_agreement_id",
"t03_inform_dep_acct.agreement_id",
"t03_inform_dep_acct.end_dt",
"t03_inform_dep_acct.inform_deposit_category",
"t03_inform_dep_acct.st_dt"},
[]string{"acct_term_temp.agreement_id:Physical",
"acct_term_temp.term:Physical",
"acct_term_temp.term_unit_cd:Physical",
"agreement_cash_pool_temp.cash_pool_group:Physical",
"agreement_cash_pool_temp.tid:Physical",
"agreement_item_temp.agreement_category_cd:Physical",
"agreement_item_temp.agreement_id:Physical",
"agreement_item_temp.agreement_mdfr:Physical",
"agreement_item_temp.agreement_stat_cd:Physical",
"agreement_item_temp.category_cd:Physical",
"agreement_item_temp.ccy_cd:Physical",
"agreement_item_temp.close_dt:Physical",
"agreement_item_temp.cur_bal:Physical",
"agreement_item_temp.item_id:Physical",
"agreement_item_temp.mature_dt:Physical",
"agreement_item_temp.open_acct_amt:Physical",
"agreement_item_temp.party_id:Physical",
"agreement_item_temp.sign_dt:Physical",
"agreement_item_temp.sign_org:Physical",
"agreement_item_temp.src_sys:Physical",
"agreement_item_temp.st_int_dt:Physical",
"s04_zmq_acc_cur.customer:Physical",
"s04_zmq_acc_cur.tid:Physical",
"t03_acct.agreement_id:Physical",
"t03_acct.dep_exchg_ind:Physical",
"t03_acct.fcy_spec_acct_id_type:Physical",
"t03_acct.sleep_acct_ind:Physical",
"t03_agreement_agt_h.agreement_id:Physical",
"t03_agreement_agt_h.agt_open_acct_verify_situati:Physical",
"t03_agreement_agt_h.agter_ident_info_category_cd:Physical",
"t03_agreement_agt_h.agter_ident_info_content:Physical",
"t03_agreement_agt_h.agter_nationality_cd:Physical",
"t03_agreement_agt_h.agter_nm:Physical",
"t03_agreement_agt_h.agter_tel:Physical",
"t03_agreement_agt_h.end_dt:Physical",
"t03_agreement_agt_h.st_dt:Physical",
"t03_agreement_int_h.agreement_id:Physical",
"t03_agreement_int_h.agreement_mdfr:Physical",
"t03_agreement_int_h.end_dt:Physical",
"t03_agreement_int_h.int_type_cd:Physical",
"t03_agreement_int_h.intr:Physical",
"t03_agreement_int_h.st_dt:Physical",
"t03_agreement_medium_rela_h.agreement_id:Physical",
"t03_agreement_medium_rela_h.agreement_medium_rela_type_cd:Physical",
"t03_agreement_medium_rela_h.end_dt:Physical",
"t03_agreement_medium_rela_h.medium_id:Physical",
"t03_agreement_medium_rela_h.st_dt:Physical",
"t03_agreement_pty_rela_h_temp.agreement_id:Physical",
"t03_agreement_pty_rela_h_temp.party_id:Physical",
"t03_agreement_rela_h_temp.agreement_id:Physical",
"t03_agreement_rela_h_temp.assoc_agreement_id:Physical",
"t03_inform_dep_acct.agreement_id:Physical",
"t03_inform_dep_acct.end_dt:Physical",
"t03_inform_dep_acct.inform_deposit_category:Physical",
"t03_inform_dep_acct.st_dt:Physical",
},
},
{
original: sql.SQLNestedSubquery,
normal: sql.SQLNestedSubqueryNormal,
referredCols: []string{"employees.job_id",
"job_history.department_id",
"job_history.job_id",
"jobs.job_id",
"jobs.min_salary",
"performance.id",
"performance.score",
referredCols: []string{"employees.job_id:Physical",
"job_history.department_id:Physical",
"job_history.job_id:Physical",
"jobs.job_id:Physical",
"jobs.min_salary:Physical",
"performance.id:Physical",
"performance.score:Physical",
},
},
}
Expand Down

0 comments on commit b77092a

Please sign in to comment.