From 9cbf6a3aacfd39e519aa717824ed0e6724b73dc4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Viktor=20S=C3=B6derqvist?= Date: Sat, 6 Nov 2021 13:56:09 +0100 Subject: [PATCH 1/6] Add --offline option and REBAR_OFFLINE environment variable If --offline or REBAR_OFFLINE=1 is given, inets is not started and all calls to httpc:request/N are prevented. If any online resource is needed, an error occurs rather than trying to fetch the online resource. --- bootstrap | 29 ++++++++++++++++++++++++----- src/rebar3.erl | 17 ++++++++++++++--- src/rebar_httpc_adapter.erl | 16 ++++++++++++---- src/rebar_prv_local_upgrade.erl | 13 +++++++++++++ src/vendored/r3_hex_http_httpc.erl | 15 ++++++++++++--- 5 files changed, 75 insertions(+), 15 deletions(-) diff --git a/bootstrap b/bootstrap index 4564ae8ff..afc818f12 100755 --- a/bootstrap +++ b/bootstrap @@ -2,14 +2,25 @@ %% -*- mode: erlang;erlang-indent-level: 4;indent-tabs-mode: nil -*- %% ex: ft=erlang ts=4 sw=4 et -main(_) -> +main(Args) -> + case lists:member("--offline", Args) of + true -> + os:putenv("REBAR_OFFLINE", "1"); + false -> + _ + end, ensure_app(crypto), ensure_app(asn1), ensure_app(public_key), - ensure_app(ssl), - inets:start(), - inets:start(httpc, [{profile, rebar}]), - set_httpc_options(), + case os:getenv("REBAR_OFFLINE") of + "1" -> + ok; + _ -> + ensure_app(ssl), + inets:start(), + inets:start(httpc, [{profile, rebar}]), + set_httpc_options() + end, %% Clear directories for builds since bootstrapping may require %% a changed structure from an older one @@ -117,6 +128,14 @@ extract(Binary) -> {ok, Contents}. request(Url) -> + case os:getenv("REBAR_OFFLINE") of + "1" -> + {error, {offline, Url}}; + _ -> + request_online(Url) + end. + +request_online(Url) -> HttpOptions = [{relaxed, true} | get_proxy_auth()], case httpc:request(get, {Url, []}, diff --git a/src/rebar3.erl b/src/rebar3.erl index a0b9266f5..5e5289ef7 100644 --- a/src/rebar3.erl +++ b/src/rebar3.erl @@ -177,6 +177,12 @@ run_aux(State, RawArgs) -> State9 = rebar_state:default(State8, rebar_state:opts(State8)), {Task, Args} = parse_args(RawArgs), + case lists:member("--offline", Args) of + true -> + os:putenv("REBAR_OFFLINE", "1"); + false -> + ok + end, State10 = rebar_state:code_paths(State9, default, code:get_path()), @@ -377,9 +383,14 @@ start_and_load_apps(Caller) -> ensure_running(crypto, Caller), ensure_running(asn1, Caller), ensure_running(public_key, Caller), - ensure_running(ssl, Caller), - ensure_running(inets, Caller), - inets:start(httpc, [{profile, rebar}]). + case os:getenv("REBAR_OFFLINE") of + "1" -> + ok; + _ -> + ensure_running(ssl, Caller), + ensure_running(inets, Caller), + inets:start(httpc, [{profile, rebar}]) + end. %% @doc Make sure a required app is running, or display an error message %% and abort if there's a problem. diff --git a/src/rebar_httpc_adapter.erl b/src/rebar_httpc_adapter.erl index b173cf6b3..26318d1ba 100644 --- a/src/rebar_httpc_adapter.erl +++ b/src/rebar_httpc_adapter.erl @@ -9,6 +9,18 @@ %%==================================================================== request(Method, URI, ReqHeaders, Body, AdapterConfig) -> + case os:getenv("REBAR_OFFLINE") of + "1" -> + {error, {offline, URI}}; + _ -> + request_online(Method, URI, ReqHeaders, Body, AdapterConfig) + end. + +%%==================================================================== +%% Internal functions +%%==================================================================== + +request_online(Method, URI, ReqHeaders, Body, AdapterConfig) -> Profile = maps:get(profile, AdapterConfig, default), Request = build_request(URI, ReqHeaders, Body), SSLOpts = [{ssl, rebar_utils:ssl_opts(URI)}], @@ -19,10 +31,6 @@ request(Method, URI, ReqHeaders, Body, AdapterConfig) -> {error, Reason} -> {error, Reason} end. -%%==================================================================== -%% Internal functions -%%==================================================================== - build_request(URI, ReqHeaders, Body) -> build_request2(binary_to_list(URI), dump_headers(ReqHeaders), Body). diff --git a/src/rebar_prv_local_upgrade.erl b/src/rebar_prv_local_upgrade.erl index 8f0d3a453..a2e99638f 100644 --- a/src/rebar_prv_local_upgrade.erl +++ b/src/rebar_prv_local_upgrade.erl @@ -120,6 +120,19 @@ etag(Path) -> ETag :: false | string(), Res :: 'error' | {ok, cached} | {ok, any(), string()}. request(Url, ETag) -> + case os:getenv("REBAR_OFFLINE") of + "1" -> + ?DEBUG("Rebar is in offline mode"), + error; + _ -> + request_online(Url, ETag) + end. + +-spec request_online(Url, ETag) -> Res when + Url :: string(), + ETag :: false | string(), + Res :: 'error' | {ok, cached} | {ok, any(), string()}. +request_online(Url, ETag) -> HttpOptions = [{ssl, rebar_utils:ssl_opts(Url)}, {relaxed, true} | rebar_utils:get_proxy_auth()], case httpc:request(get, {Url, [{"if-none-match", "\"" ++ ETag ++ "\""} diff --git a/src/vendored/r3_hex_http_httpc.erl b/src/vendored/r3_hex_http_httpc.erl index 79a354779..2ca079ebe 100644 --- a/src/vendored/r3_hex_http_httpc.erl +++ b/src/vendored/r3_hex_http_httpc.erl @@ -11,6 +11,18 @@ %%==================================================================== request(Method, URI, ReqHeaders, Body, AdapterConfig) -> + case os:getenv("REBAR_OFFLINE") of + "1" -> + {error, {offline, URI}}; + _ -> + request_online(Method, URI, ReqHeaders, Body, AdapterConfig) + end. + +%%==================================================================== +%% Internal functions +%%==================================================================== + +request_online(Method, URI, ReqHeaders, Body, AdapterConfig) -> Profile = maps:get(profile, AdapterConfig, default), Request = build_request(URI, ReqHeaders, Body), case httpc:request(Method, Request, [], [{body_format, binary}], Profile) of @@ -20,9 +32,6 @@ request(Method, URI, ReqHeaders, Body, AdapterConfig) -> {error, Reason} -> {error, Reason} end. -%%==================================================================== -%% Internal functions -%%==================================================================== build_request(URI, ReqHeaders, Body) -> build_request2(binary_to_list(URI), dump_headers(ReqHeaders), Body). From 3f249fde4cb28bfea06d66c2a6f84d28ab94eabd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Viktor=20S=C3=B6derqvist?= Date: Fri, 12 Nov 2021 09:40:58 +0100 Subject: [PATCH 2/6] Fixup: typos --- bootstrap | 2 +- src/rebar_prv_local_upgrade.erl | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/bootstrap b/bootstrap index afc818f12..96b4387c5 100755 --- a/bootstrap +++ b/bootstrap @@ -7,7 +7,7 @@ main(Args) -> true -> os:putenv("REBAR_OFFLINE", "1"); false -> - _ + ok end, ensure_app(crypto), ensure_app(asn1), diff --git a/src/rebar_prv_local_upgrade.erl b/src/rebar_prv_local_upgrade.erl index a2e99638f..74d198da6 100644 --- a/src/rebar_prv_local_upgrade.erl +++ b/src/rebar_prv_local_upgrade.erl @@ -122,7 +122,7 @@ etag(Path) -> request(Url, ETag) -> case os:getenv("REBAR_OFFLINE") of "1" -> - ?DEBUG("Rebar is in offline mode"), + ?DEBUG("Rebar is in offline mode", []), error; _ -> request_online(Url, ETag) From 81c889444f658be9ae5e7d89f70230425bc10e4c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Viktor=20S=C3=B6derqvist?= Date: Mon, 2 May 2022 10:13:09 +0200 Subject: [PATCH 3/6] Intercept download and needs_update in rebar_fetch --- src/rebar_fetch.erl | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/src/rebar_fetch.erl b/src/rebar_fetch.erl index d32a42312..7e5d52823 100644 --- a/src/rebar_fetch.erl +++ b/src/rebar_fetch.erl @@ -50,6 +50,14 @@ download_source(AppInfo, State) -> end. download_source_(AppInfo, State) -> + case os:getenv("REBAR_OFFLINE") of + "1" -> + {error, {?MODULE, offline}}; + _ -> + download_source_online(AppInfo, State) + end. + +download_source_online(AppInfo, State) -> AppDir = rebar_app_info:dir(AppInfo), TmpDir = ec_file:insecure_mkdtemp(), AppDir1 = rebar_utils:to_list(AppDir), @@ -68,6 +76,17 @@ download_source_(AppInfo, State) -> -spec needs_update(rebar_app_info:t(), rebar_state:t()) -> boolean() | {error, string()}. needs_update(AppInfo, State) -> + case os:getenv("REBAR_OFFLINE") of + "1" -> + ?DEBUG("Can't check if dependency needs updates in offline mode", []), + false; + _ -> + needs_update_online(AppInfo, State) + end. + +-spec needs_update_online(rebar_app_info:t(), rebar_state:t()) + -> boolean() | {error, string()}. +needs_update_online(AppInfo, State) -> try rebar_resource_v2:needs_update(AppInfo, State) catch From a74054c37eb47bceb0dc2fefccbdbc6e009f0ca5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Viktor=20S=C3=B6derqvist?= Date: Mon, 2 May 2022 10:19:12 +0200 Subject: [PATCH 4/6] Revert changes in src/vendored/ --- src/vendored/r3_hex_http_httpc.erl | 15 +++------------ 1 file changed, 3 insertions(+), 12 deletions(-) diff --git a/src/vendored/r3_hex_http_httpc.erl b/src/vendored/r3_hex_http_httpc.erl index 2ca079ebe..79a354779 100644 --- a/src/vendored/r3_hex_http_httpc.erl +++ b/src/vendored/r3_hex_http_httpc.erl @@ -11,18 +11,6 @@ %%==================================================================== request(Method, URI, ReqHeaders, Body, AdapterConfig) -> - case os:getenv("REBAR_OFFLINE") of - "1" -> - {error, {offline, URI}}; - _ -> - request_online(Method, URI, ReqHeaders, Body, AdapterConfig) - end. - -%%==================================================================== -%% Internal functions -%%==================================================================== - -request_online(Method, URI, ReqHeaders, Body, AdapterConfig) -> Profile = maps:get(profile, AdapterConfig, default), Request = build_request(URI, ReqHeaders, Body), case httpc:request(Method, Request, [], [{body_format, binary}], Profile) of @@ -32,6 +20,9 @@ request_online(Method, URI, ReqHeaders, Body, AdapterConfig) -> {error, Reason} -> {error, Reason} end. +%%==================================================================== +%% Internal functions +%%==================================================================== build_request(URI, ReqHeaders, Body) -> build_request2(binary_to_list(URI), dump_headers(ReqHeaders), Body). From 2d5769245887cf3eb851dac911006614e494c0f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Viktor=20S=C3=B6derqvist?= Date: Fri, 27 May 2022 09:55:26 +0200 Subject: [PATCH 5/6] Store offline in state and use it where possible --- src/rebar3.erl | 26 +++++++++++++++----------- src/rebar_fetch.erl | 12 ++++++------ 2 files changed, 21 insertions(+), 17 deletions(-) diff --git a/src/rebar3.erl b/src/rebar3.erl index 5e5289ef7..ea55da591 100644 --- a/src/rebar3.erl +++ b/src/rebar3.erl @@ -177,23 +177,27 @@ run_aux(State, RawArgs) -> State9 = rebar_state:default(State8, rebar_state:opts(State8)), {Task, Args} = parse_args(RawArgs), - case lists:member("--offline", Args) of - true -> - os:putenv("REBAR_OFFLINE", "1"); - false -> - ok - end, + Offline = case lists:member("--offline", Args) of + true -> + %% We store this redundantly in env, because some APIs + %% don't get a state. + os:putenv("REBAR_OFFLINE", "1"), + true; + false -> + os:getenv("REBAR_OFFLINE") =:= "1" + end, + State10 = rebar_state:set(State9, offline, Offline); - State10 = rebar_state:code_paths(State9, default, code:get_path()), + State11 = rebar_state:code_paths(State10, default, code:get_path()), - case rebar_core:init_command(rebar_state:command_args(State10, Args), Task) of - {ok, State11} -> + case rebar_core:init_command(rebar_state:command_args(State11, Args), Task) of + {ok, State12} -> case rebar_state:get(State11, caller, command_line) of api -> rebar_paths:unset_paths([deps, plugins], State11), - {ok, State11}; + {ok, State12}; _ -> - {ok, State11} + {ok, State12} end; Other -> Other diff --git a/src/rebar_fetch.erl b/src/rebar_fetch.erl index 7e5d52823..28753a308 100644 --- a/src/rebar_fetch.erl +++ b/src/rebar_fetch.erl @@ -50,10 +50,10 @@ download_source(AppInfo, State) -> end. download_source_(AppInfo, State) -> - case os:getenv("REBAR_OFFLINE") of - "1" -> + case rebar_state:get(State, offline, false) of + true -> {error, {?MODULE, offline}}; - _ -> + false -> download_source_online(AppInfo, State) end. @@ -76,11 +76,11 @@ download_source_online(AppInfo, State) -> -spec needs_update(rebar_app_info:t(), rebar_state:t()) -> boolean() | {error, string()}. needs_update(AppInfo, State) -> - case os:getenv("REBAR_OFFLINE") of - "1" -> + case rebar_state:get(State, offline, false) of + true -> ?DEBUG("Can't check if dependency needs updates in offline mode", []), false; - _ -> + false -> needs_update_online(AppInfo, State) end. From 4f3bb005655a5040c22603e3a11937c657dc7991 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Viktor=20S=C3=B6derqvist?= Date: Fri, 27 May 2022 17:58:07 +0200 Subject: [PATCH 6/6] Fix syntax error typo --- src/rebar3.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/rebar3.erl b/src/rebar3.erl index ea55da591..3e80ea0c5 100644 --- a/src/rebar3.erl +++ b/src/rebar3.erl @@ -186,7 +186,7 @@ run_aux(State, RawArgs) -> false -> os:getenv("REBAR_OFFLINE") =:= "1" end, - State10 = rebar_state:set(State9, offline, Offline); + State10 = rebar_state:set(State9, offline, Offline), State11 = rebar_state:code_paths(State10, default, code:get_path()),