diff --git a/README.md b/README.md index 62c644e..17c49ad 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,8 @@ # go-sql-lineage Pure golang SQL lineage analysis toolkit +1. SQL:2011 based on [postgresql-parser](https://github.com/auxten/postgresql-parser) +1. Very complex SQL tested, see [SQLs](test/sql/sqls.go), 65.7% test coverage ## Quick Start diff --git a/lineage/from_test.go b/lineage/from_test.go index c367e2b..c759752 100644 --- a/lineage/from_test.go +++ b/lineage/from_test.go @@ -37,7 +37,7 @@ func TestGetProvidedColumns(t *testing.T) { }, }, sql.SQLNestedSubquery, - "[employees.job_id employees.salary p.id p.score maxavg.max(myavg) maxavg.job_id]", + "[employees.job_id:Physical employees.salary:Physical p.id:Physical p.score:Physical maxavg.max(myavg):Unknown maxavg.job_id:Unknown]", "[employees p maxavg]", }, { @@ -53,7 +53,7 @@ func TestGetProvidedColumns(t *testing.T) { }, }, sql.SQLUnion, - "[type contactname city country]", + "[type:Unknown contactname:Physical city:Physical country:Physical]", "[]", }, { @@ -96,7 +96,7 @@ func TestGetProvidedColumns(t *testing.T) { }, }, sql.SQLmj, - "[clerk.rowid clerk.oper_no clerk.oper_name clerk.cert_no clerk.oper_no cifo.rowid cifo.cert_no cifo.cust_no basic.cust_no basic.acct_no basic.unnecessary acct.sub_code acct.acct_no acct.acct_bal]", + "[clerk.rowid:Physical clerk.oper_no:Physical clerk.oper_name:Physical clerk.cert_no:Physical clerk.oper_no:Physical cifo.rowid:Physical cifo.cert_no:Physical cifo.cust_no:Physical basic.cust_no:Physical basic.acct_no:Physical basic.unnecessary:Physical acct.sub_code:Physical acct.acct_no:Physical acct.acct_bal:Physical]", "[clerk cifo basic acct]", }, } diff --git a/lineage/normalize_test.go b/lineage/normalize_test.go index 5ed07ef..90aedab 100644 --- a/lineage/normalize_test.go +++ b/lineage/normalize_test.go @@ -75,12 +75,12 @@ func TestNormalizeASTWithoutInputSchema(t *testing.T) { { sql.SQLMultiFrom, sql.SQLMultiFromNormal, - []string{"a", "b", "ct4", "e", "t31.c", "t31.d"}, + []string{"a:Unknown", "b:Unknown", "ct4:Unknown", "e:Unknown", "t31.c:Physical", "t31.d:Physical"}, }, { sql.SQLmj, sql.SQLmjNormal, - []string{"comc_clerk.cert_no", "comc_clerk.oper_name", "comc_clerk.oper_no", "comr_cifbinfo.cert_no", "comr_cifbinfo.cust_no", "savb_acctinfo_chk.acct_bal", "savb_acctinfo_chk.acct_no", "savb_acctinfo_chk.sub_code", "savb_basicinfo.acct_no", "savb_basicinfo.cust_no", "savb_basicinfo.unnecessary"}, + []string{"comc_clerk.cert_no:Physical", "comc_clerk.oper_name:Physical", "comc_clerk.oper_no:Physical", "comr_cifbinfo.cert_no:Physical", "comr_cifbinfo.cust_no:Physical", "savb_acctinfo_chk.acct_bal:Physical", "savb_acctinfo_chk.acct_no:Physical", "savb_acctinfo_chk.sub_code:Physical", "savb_basicinfo.acct_no:Physical", "savb_basicinfo.cust_no:Physical", "savb_basicinfo.unnecessary:Physical"}, }, { `select marr @@ -114,77 +114,78 @@ ORDER BY table1.sex DESC LIMIT 1`, - []string{"root_loan_mock_v4.age", "root_loan_mock_v4.label", "root_loan_mock_v4.marr_stat_cd", "table1.sex"}, + []string{"root_loan_mock_v4.age:Physical", "root_loan_mock_v4.label:Physical", "root_loan_mock_v4.marr_stat_cd:Physical", "table1.sex:Physical"}, }, { sql.SQL246, sql.SQL246Normal, - []string{"acct_term_temp.agreement_id", - "acct_term_temp.term", - "acct_term_temp.term_unit_cd", - "agreement_cash_pool_temp.cash_pool_group", - "agreement_cash_pool_temp.tid", - "agreement_item_temp.agreement_category_cd", - "agreement_item_temp.agreement_id", - "agreement_item_temp.agreement_mdfr", - "agreement_item_temp.agreement_stat_cd", - "agreement_item_temp.category_cd", - "agreement_item_temp.ccy_cd", - "agreement_item_temp.close_dt", - "agreement_item_temp.cur_bal", - "agreement_item_temp.item_id", - "agreement_item_temp.mature_dt", - "agreement_item_temp.open_acct_amt", - "agreement_item_temp.party_id", - "agreement_item_temp.sign_dt", - "agreement_item_temp.sign_org", - "agreement_item_temp.src_sys", - "agreement_item_temp.st_int_dt", - "s04_zmq_acc_cur.customer", - "s04_zmq_acc_cur.tid", - "t03_acct.agreement_id", - "t03_acct.dep_exchg_ind", - "t03_acct.fcy_spec_acct_id_type", - "t03_acct.sleep_acct_ind", - "t03_agreement_agt_h.agreement_id", - "t03_agreement_agt_h.agt_open_acct_verify_situati", - "t03_agreement_agt_h.agter_ident_info_category_cd", - "t03_agreement_agt_h.agter_ident_info_content", - "t03_agreement_agt_h.agter_nationality_cd", - "t03_agreement_agt_h.agter_nm", - "t03_agreement_agt_h.agter_tel", - "t03_agreement_agt_h.end_dt", - "t03_agreement_agt_h.st_dt", - "t03_agreement_int_h.agreement_id", - "t03_agreement_int_h.agreement_mdfr", - "t03_agreement_int_h.end_dt", - "t03_agreement_int_h.int_type_cd", - "t03_agreement_int_h.intr", - "t03_agreement_int_h.st_dt", - "t03_agreement_medium_rela_h.agreement_id", - "t03_agreement_medium_rela_h.agreement_medium_rela_type_cd", - "t03_agreement_medium_rela_h.end_dt", - "t03_agreement_medium_rela_h.medium_id", - "t03_agreement_medium_rela_h.st_dt", - "t03_agreement_pty_rela_h_temp.agreement_id", - "t03_agreement_pty_rela_h_temp.party_id", - "t03_agreement_rela_h_temp.agreement_id", - "t03_agreement_rela_h_temp.assoc_agreement_id", - "t03_inform_dep_acct.agreement_id", - "t03_inform_dep_acct.end_dt", - "t03_inform_dep_acct.inform_deposit_category", - "t03_inform_dep_acct.st_dt"}, + []string{"acct_term_temp.agreement_id:Physical", + "acct_term_temp.term:Physical", + "acct_term_temp.term_unit_cd:Physical", + "agreement_cash_pool_temp.cash_pool_group:Physical", + "agreement_cash_pool_temp.tid:Physical", + "agreement_item_temp.agreement_category_cd:Physical", + "agreement_item_temp.agreement_id:Physical", + "agreement_item_temp.agreement_mdfr:Physical", + "agreement_item_temp.agreement_stat_cd:Physical", + "agreement_item_temp.category_cd:Physical", + "agreement_item_temp.ccy_cd:Physical", + "agreement_item_temp.close_dt:Physical", + "agreement_item_temp.cur_bal:Physical", + "agreement_item_temp.item_id:Physical", + "agreement_item_temp.mature_dt:Physical", + "agreement_item_temp.open_acct_amt:Physical", + "agreement_item_temp.party_id:Physical", + "agreement_item_temp.sign_dt:Physical", + "agreement_item_temp.sign_org:Physical", + "agreement_item_temp.src_sys:Physical", + "agreement_item_temp.st_int_dt:Physical", + "s04_zmq_acc_cur.customer:Physical", + "s04_zmq_acc_cur.tid:Physical", + "t03_acct.agreement_id:Physical", + "t03_acct.dep_exchg_ind:Physical", + "t03_acct.fcy_spec_acct_id_type:Physical", + "t03_acct.sleep_acct_ind:Physical", + "t03_agreement_agt_h.agreement_id:Physical", + "t03_agreement_agt_h.agt_open_acct_verify_situati:Physical", + "t03_agreement_agt_h.agter_ident_info_category_cd:Physical", + "t03_agreement_agt_h.agter_ident_info_content:Physical", + "t03_agreement_agt_h.agter_nationality_cd:Physical", + "t03_agreement_agt_h.agter_nm:Physical", + "t03_agreement_agt_h.agter_tel:Physical", + "t03_agreement_agt_h.end_dt:Physical", + "t03_agreement_agt_h.st_dt:Physical", + "t03_agreement_int_h.agreement_id:Physical", + "t03_agreement_int_h.agreement_mdfr:Physical", + "t03_agreement_int_h.end_dt:Physical", + "t03_agreement_int_h.int_type_cd:Physical", + "t03_agreement_int_h.intr:Physical", + "t03_agreement_int_h.st_dt:Physical", + "t03_agreement_medium_rela_h.agreement_id:Physical", + "t03_agreement_medium_rela_h.agreement_medium_rela_type_cd:Physical", + "t03_agreement_medium_rela_h.end_dt:Physical", + "t03_agreement_medium_rela_h.medium_id:Physical", + "t03_agreement_medium_rela_h.st_dt:Physical", + "t03_agreement_pty_rela_h_temp.agreement_id:Physical", + "t03_agreement_pty_rela_h_temp.party_id:Physical", + "t03_agreement_rela_h_temp.agreement_id:Physical", + "t03_agreement_rela_h_temp.assoc_agreement_id:Physical", + "t03_inform_dep_acct.agreement_id:Physical", + "t03_inform_dep_acct.end_dt:Physical", + "t03_inform_dep_acct.inform_deposit_category:Physical", + "t03_inform_dep_acct.st_dt:Physical", + }, }, { original: sql.SQLNestedSubquery, normal: sql.SQLNestedSubqueryNormal, - referredCols: []string{"employees.job_id", - "job_history.department_id", - "job_history.job_id", - "jobs.job_id", - "jobs.min_salary", - "performance.id", - "performance.score", + referredCols: []string{"employees.job_id:Physical", + "job_history.department_id:Physical", + "job_history.job_id:Physical", + "jobs.job_id:Physical", + "jobs.min_salary:Physical", + "performance.id:Physical", + "performance.score:Physical", }, }, }