Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: validate the subordinate and check reference when deleting and adding… #8219

Closed
Closed
Show file tree
Hide file tree
Changes from 13 commits
Commits
Show all changes
58 commits
Select commit Hold shift + click to select a range
5d10f41
validate the subordinate and check reference when deleting a stream …
Oct 31, 2022
734b43f
validate the subordinate and check reference when deleting a stream …
Oct 31, 2022
4f13eed
validate the subordinate and check reference when deleting a stream …
Oct 31, 2022
f0f6ff9
validate the subordinate and check reference when deleting a stream …
Oct 31, 2022
9168493
add test cases to check res.refer
Nov 1, 2022
cc276f7
fix cache in refer
Nov 1, 2022
7db5790
fix some bugs
Nov 2, 2022
34f79d2
fix (ci check fail)
Nov 3, 2022
4dde2e9
fix (ci check fail)
Nov 3, 2022
cba2083
format
Nov 4, 2022
94cdf53
another imp no etcd
Nov 5, 2022
b6cc11e
another imp no etcd
Nov 5, 2022
bb5a2a5
fix errors in CI stage
Nov 6, 2022
f04b2c5
style:
Nov 7, 2022
475007e
revise tablepool
Nov 7, 2022
527110c
Update stream_routes.lua
biakewe Nov 8, 2022
9f08dbb
drop trailing whitespace
Nov 9, 2022
02709ab
fix test case
Nov 9, 2022
4ecd49f
fix test case
Nov 9, 2022
3c7d4d3
fix test case
Nov 9, 2022
dee8669
fix test case
Nov 9, 2022
5aec5cb
fix APISIX.PM
Nov 10, 2022
1ae67e6
fix APISIX.PM
Nov 10, 2022
9f56771
debug
Nov 10, 2022
71c0f89
Get stream_routes from etcd directly
Nov 11, 2022
2fc55c9
Get stream_routes from etcd directly
Nov 11, 2022
e1894f8
Update stream-routes.t
biakewe Nov 11, 2022
63f4d7b
update
Nov 14, 2022
775fff0
Merge branch 'check-the-subordinate-relationship' of https://github.c…
Nov 14, 2022
f322e4f
Validate if the protocol matches the subordinate only if the stream r…
Nov 14, 2022
b5dba07
Validate if the protocol matches the subordinate only if the stream r…
Nov 14, 2022
d69ce76
Validate if the protocol matches the subordinate only if the stream r…
Nov 14, 2022
f6d98e2
fix
Nov 14, 2022
a64bb1e
fix
Nov 14, 2022
2854be0
fix
Nov 14, 2022
ed245b5
fix
Nov 14, 2022
c15e3e6
fix
Nov 14, 2022
49609a7
fix
Nov 14, 2022
3205a75
fix
Nov 14, 2022
0cbd892
debug test case
Nov 14, 2022
573578d
debug test case
Nov 14, 2022
09e34e7
debug test case
Nov 14, 2022
8ab326c
update
Nov 14, 2022
9b1533d
fix indent
Nov 15, 2022
63ff87f
no-etcd implementation
Nov 16, 2022
11315d8
no-etcd implementation
Nov 16, 2022
23d4ace
no-etcd implementation
Nov 16, 2022
5f4c836
update test case
Nov 17, 2022
aafaaf2
update test case
Nov 17, 2022
be47504
update init
Nov 17, 2022
9289dd7
update init
Nov 17, 2022
3f85069
update init
Nov 17, 2022
d9f3a0a
update admin.init
Nov 18, 2022
dbd0e02
update admin.init
Nov 18, 2022
1517c85
update
Nov 21, 2022
cd17fcc
Merge branch 'master' into check-the-subordinate-relationship
biakewe Nov 23, 2022
401ba10
update
Nov 24, 2022
2c0a1b1
Merge branch 'check-the-subordinate-relationship' of https://github.c…
Nov 24, 2022
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
31 changes: 31 additions & 0 deletions apisix/admin/stream_routes.lua
Original file line number Diff line number Diff line change
Expand Up @@ -16,15 +16,36 @@
--
local core = require("apisix.core")
local utils = require("apisix.admin.utils")
local config_util = require("apisix.core.config_util")
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggested change
local config_util = require("apisix.core.config_util")
local iterate_values = require("apisix.core.config_util").iterate_values

local routes = require("apisix.stream.router.ip_port").routes
local stream_route_checker = require("apisix.stream.router.ip_port").stream_route_checker
local tostring = tostring
local string = string
local table = table


local _M = {
version = 0.1,
need_v3_filter = true,
}

local function check_router_refer(items, id)
local refer_list = {}
for _, item in config_util.iterate_values(items) do
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggested change
for _, item in config_util.iterate_values(items) do
for _, item in iterate_values(items) do

if item.value == nil then
goto CONTINUE
end
local r_id = string.gsub(item["key"], "/", "_")
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

can we replace string.gsub with ngx.re.gsub?

local route = item.value
if route.protocol and route.protocol.superior_id and route.protocol.superior_id == id then
table.insert(refer_list,r_id)
end
::CONTINUE::
end
return refer_list
end



local function check_conf(id, conf, need_id)
if not conf then
Expand Down Expand Up @@ -142,8 +163,18 @@ function _M.delete(id)
return 400, {error_msg = "missing stream route id"}
end

local items,_ = routes()
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggested change
local items,_ = routes()
local items, _ = routes()

local key = "/stream_routes/" .. id
-- core.log.info("key: ", key)
local refer_list = {}
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggested change
local refer_list = {}
local refer_list

we don't need to init refer_list here? it would be inited in check_router_refer.

if items ~= nil then
refer_list=check_router_refer(items,id)
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggested change
refer_list=check_router_refer(items,id)
refer_list = check_router_refer(items,id)

end
if #refer_list >0 then
local warn_message = key.." is referred by "..table.concat(refer_list,";;")
return 400,warn_message
end
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

After thinking about it for a while, it seems that the refer_list lifecycle is only used here. Do we use core.tablepool to fetch and release refer_list?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thanks for your comments about core.tablepool.


local res, err = core.etcd.delete(key)
if not res then
core.log.error("failed to delete stream route[", key, "]: ", err)
Expand Down
1 change: 1 addition & 0 deletions apisix/cli/ngx_tpl.lua
Original file line number Diff line number Diff line change
Expand Up @@ -447,6 +447,7 @@ http {

init_worker_by_lua_block {
apisix.http_init_worker()
apisix.stream_init_worker()
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

we don't need this

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I want to explain my code design :
This is using logic about stream_routes inside admin_api(/apisix/admin). Get stream_routes from apisix.stream.router.ip_port.routes() and apisix.stream.router.ip_port.routes() depends on stream_init_worker(). So under the scope of admin_api , I have to add stream_init_worker in advance。

}

exit_worker_by_lua_block {
Expand Down
25 changes: 24 additions & 1 deletion apisix/stream/router/ip_port.lua
Original file line number Diff line number Diff line change
Expand Up @@ -63,23 +63,46 @@ local create_router
do
local sni_to_items = {}
local tls_routes = {}
local routeid_to_protocols = {}

function create_router(items)
local tls_routes_idx = 1
local other_routes_idx = 1
core.table.clear(tls_routes)
core.table.clear(other_routes)
core.table.clear(sni_to_items)
core.table.clear(routeid_to_protocols)

for _, item in config_util.iterate_values(items) do
if item.value == nil then
goto CONTINUE
end
local route = item.value
if route.protocol then
routeid_to_protocols[item.key]=route.protocol.name
else
routeid_to_protocols[item.key]="No-Protocol"
end
::CONTINUE::
end

for _, item in config_util.iterate_values(items) do
if item.value == nil then
goto CONTINUE
end
local route = item.value
if route.protocol and route.protocol.superior_id then
-- subordinate route won't be matched in the entry
-- TODO: check the subordinate relationship in the Admin API
local key="/apisix/stream_routes/"..route.protocol.superior_id
if routeid_to_protocols[key] == nil then
core.log.warn("There is not exist stream_route: "..key)
elseif routeid_to_protocols[key] == "No-Protocol" then
core.log.warn("The stream_route: "..key.." may lacks procotol configuration")
elseif routeid_to_protocols[key] == route.protocol.name then
goto CONTINUE
else
core.log.warn("Different between stream_route:"..item.key.." and "..key)
end
goto CONTINUE
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It seems we don't need to modify here? I see that the test cases don't cover the changes here.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes, no action here, just warn logging。In order to remind users, do you think it's ok to keep this?

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I would prefer to check this when adding the stream route and return it if it is wrong.

In addition, two for loops are used here, which is even less efficient.

Copy link
Contributor Author

@biakewe biakewe Nov 14, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Add a logic that checking this when adding the stream route , there are two situations to judge:
1 validate if the stream route with superior id exists
The user needs to ensure the order when creating routes in batches --> "super is in front of sub "
2 validate if the stream route with superior id exists and its protocol matches the subordinate
Validate if the protocol matches the subordinate only if the stream route with superior id exists。

So I would add a logic that Validate if the protocol matches the subordinate only if the stream route with superior id exists。Do you think it's ok?

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Add a logic that checking this when adding the stream route , there are two situations to judge: 1 validate if the stream route with superior id exists The user needs to ensure the order when creating routes in batches --> "super is in front of sub " 2 validate if the stream route with superior id exists and its protocol matches the subordinate Validate if the protocol matches the subordinate only if the stream route with superior id exists。

So I would add a logic that Validate if the protocol matches the subordinate only if the stream route with superior id exists。Do you think it's ok?

I have a suggestion that in this PR we only finish checking refer when deleting the stream route.

You can open a new issue and PR to do what you said.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Let's keep it simple and clear enough in this PR.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

ok

end

Expand Down
1 change: 1 addition & 0 deletions t/APISIX.pm
Original file line number Diff line number Diff line change
Expand Up @@ -603,6 +603,7 @@ _EOC_

init_worker_by_lua_block {
require("apisix").http_init_worker()
require("apisix").stream_init_worker()
$extra_init_worker_by_lua
}

Expand Down
83 changes: 83 additions & 0 deletions t/admin/stream-routes.t
Original file line number Diff line number Diff line change
Expand Up @@ -639,3 +639,86 @@ passed
{"error_msg":"property \"faults\" validation failed: wrong type: expected array, got string"}
--- no_error_log
[error]

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

need three blank lines between test cases

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

ok



=== TEST 17: put reference route + delete
xrpc:
protocols:
- name: pingpong
--- config
location /t {
content_by_lua_block {
local t = require("lib.test_admin").test
local etcd = require("apisix.core.etcd")
local code, body = t('/apisix/admin/stream_routes/12',
ngx.HTTP_PUT,
[[{
"remote_addr": "127.0.0.1",
"desc": "test-desc",
"upstream": {
"nodes": {
"127.0.0.1:8080": 1
},
"type": "roundrobin"
},
"protocol": {
"name": "pingpong",
"superior_id": "1",
"conf": {
"faults": [{
"commands": ["get", "ping"],
"delay": 5
}]
}
"desc": "new route"
}]],
[[{
"value": {
"remote_addr": "127.0.0.1",
"desc": "test-desc",
"upstream": {
"nodes": {
"127.0.0.1:8080": 1
},
"type": "roundrobin"
},
"protocol": {
"name": "pingpong",
"superior_id": "1",
"conf": {
"faults": [{
"commands": ["get", "ping"],
"delay": 5
}]
}
"desc": "new route"
},
"key": "/apisix/stream_routes/12"
}]]
)
ngx.status = code
ngx.say(body)
local res = assert(etcd.get('/stream_routes/12'))
local create_time = res.body.node.value.create_time
assert(create_time ~= nil, "create_time is nil")
local update_time = res.body.node.value.update_time
assert(update_time ~= nil, "update_time is nil")
if code ~= 200 then
ngx.status = code
ngx.say(message)
return
end
ngx.say("[push] code: ", code, " message: ", message)
local id = string.sub(res.key, #"/apisix/stream_routes/" + 1)
code, message = t('/apisix/admin/stream_routes/' .. id, ngx.HTTP_DELETE)
ngx.say("[delete] code: ", code, " message: ", message)
}
}
--- request
GET /t
--- response_body
[push] code: 200 message: passed
[delete] code: 400 message: /stream_routes/1 is referred by _apisix_stream_routes_12
--- no_error_log
[error]