[TT-3158] Fix for quantity match expressions on method transform paths

[ghost]

Description

Fixes the behavior described in #3547

Related Issue

#3547

Motivation and Context

Before this change, a path like /my/path/(?:[0-9a-fA-F]{8})$ would be treated as /my/path/(?:[0-9a-fA-F]([^/]*))$, which causes unexpected behavior on url_rewrites, method_transforms and transform_headers in extended_paths

How This Has Been Tested

In order to test this the bellow API can be loaded (make sure to create a new webhoo on the target_url).

The request

GET http://127.0.0.1:8080/base/v1/resource/abcd12345
x-remove-header: remove-value
x-test-header: test-value

should behave the same as this request

GET http://127.0.0.1:8080/base/v1/resource/test1234
x-remove-header: remove-value
x-test-header: test-value

API Definition

{
  "name": "API - v1",
  "slug": "base_v1",
  "listen_port": 0,
  "protocol": "http",
  "enable_proxy_protocol": false,
  "api_id": "base_v1",
  "org_id": "",
  "use_keyless": true,
  "use_oauth2": false,
  "use_openid": false,
  "openid_options": {
    "providers": null,
    "segregate_by_client": false
  },
  "oauth_meta": {
    "allowed_access_types": null,
    "allowed_authorize_types": null,
    "auth_login_redirect": ""
  },
  "auth": {
    "use_param": false,
    "param_name": "",
    "use_cookie": false,
    "cookie_name": "",
    "auth_header_name": "",
    "use_certificate": false,
    "validate_signature": false,
    "signature": {
      "algorithm": "",
      "header": "",
      "secret": "",
      "allowed_clock_skew": 0,
      "error_code": 0,
      "error_message": ""
    }
  },
  "auth_configs": null,
  "use_basic_auth": false,
  "basic_auth": {
    "disable_caching": false,
    "cache_ttl": 0,
    "extract_from_body": false,
    "body_user_regexp": "",
    "body_password_regexp": ""
  },
  "use_mutual_tls_auth": false,
  "client_certificates": null,
  "upstream_certificates": null,
  "pinned_public_keys": null,
  "enable_jwt": false,
  "use_standard_auth": false,
  "use_go_plugin_auth": false,
  "enable_coprocess_auth": false,
  "jwt_signing_method": "",
  "jwt_source": "",
  "jwt_identity_base_field": "",
  "jwt_client_base_field": "",
  "jwt_policy_field_name": "",
  "jwt_default_policies": null,
  "jwt_issued_at_validation_skew": 0,
  "jwt_expires_at_validation_skew": 0,
  "jwt_not_before_validation_skew": 0,
  "jwt_skip_kid": false,
  "jwt_scope_to_policy_mapping": null,
  "jwt_scope_claim_name": "",
  "notifications": {
    "shared_secret": "",
    "oauth_on_keychange_url": ""
  },
  "enable_signature_checking": false,
  "hmac_allowed_clock_skew": 0,
  "hmac_allowed_algorithms": null,
  "request_signing": {
    "is_enabled": false,
    "secret": "",
    "key_id": "",
    "algorithm": "",
    "header_list": null,
    "certificate_id": "",
    "signature_header": ""
  },
  "base_identity_provided_by": "",
  "definition": {
    "location": "url",
    "key": "",
    "strip_path": false
  },
  "version_data": {
    "not_versioned": true,
    "default_version": "",
    "versions": {
      "Default": {
        "name": "Default",
        "expires": "",
        "paths": {
          "ignored": null,
          "white_list": null,
          "black_list": null
        },
        "use_extended_paths": true,
        "extended_paths": {
          "transform_headers": [
            {
              "delete_headers": [
                "x-remove-header"
              ],
              "add_headers": {
                "x-add-header": "add-value"
              },
              "path": "/resource/((?:[0-9a-fA-F]{8}))$",
              "method": "GET",
              "act_on": false
            }
          ],
          "url_rewrites": [
            {
              "path": "/resource/((?:[0-9a-fA-F]{8}))$",
              "method": "GET",
              "match_pattern": "^/base/v1/resource/((?:[0-9a-fA-F]{8}))$",
              "rewrite_to": "https://httpbin.org/post?id=$1",
              "triggers": null
            }
          ],
          "method_transforms": [
            {
              "path": "/resource/((?:[0-9a-fA-F]{8}))$",
              "method": "GET",
              "to_method": "POST"
            }
          ]
        },
        "global_headers": null,
        "global_headers_remove": null,
        "global_response_headers": null,
        "global_response_headers_remove": null,
        "ignore_endpoint_case": false,
        "global_size_limit": 0,
        "override_target": ""
      }
    }
  },
  "uptime_tests": {
    "check_list": null,
    "config": {
      "expire_utime_after": 0,
      "service_discovery": {
        "use_discovery_service": false,
        "query_endpoint": "",
        "use_nested_query": false,
        "parent_data_path": "",
        "data_path": "",
        "port_data_path": "",
        "target_path": "",
        "use_target_list": false,
        "cache_timeout": 0,
        "endpoint_returns_list": false
      },
      "recheck_wait": 0
    }
  },
  "proxy": {
    "preserve_host_header": false,
    "listen_path": "/base/v1/",
    "target_url": "https://webhook.site/0c3d0806-3840-4c11-8677-3dd54307d641",
    "disable_strip_slash": false,
    "strip_listen_path": false,
    "enable_load_balancing": false,
    "target_list": null,
    "check_host_against_uptime_tests": false,
    "service_discovery": {
      "use_discovery_service": false,
      "query_endpoint": "",
      "use_nested_query": false,
      "parent_data_path": "",
      "data_path": "",
      "port_data_path": "",
      "target_path": "",
      "use_target_list": false,
      "cache_timeout": 0,
      "endpoint_returns_list": false
    },
    "transport": {
      "ssl_insecure_skip_verify": false,
      "ssl_ciphers": null,
      "ssl_min_version": 0,
      "ssl_force_common_name_check": false,
      "proxy_url": ""
    }
  },
  "disable_rate_limit": false,
  "disable_quota": false,
  "custom_middleware": {
    "pre": [],
    "post": null,
    "post_key_auth": null,
    "auth_check": {
      "name": "",
      "path": "",
      "require_session": false,
      "raw_body_only": false
    },
    "response": null,
    "driver": "",
    "id_extractor": {
      "extract_from": "",
      "extract_with": "",
      "extractor_config": null
    }
  },
  "custom_middleware_bundle": "",
  "cache_options": {
    "cache_timeout": 0,
    "enable_cache": false,
    "cache_all_safe_requests": false,
    "cache_response_codes": null,
    "enable_upstream_cache_control": false,
    "cache_control_ttl_header": "",
    "cache_by_headers": null
  },
  "session_lifetime": 0,
  "active": false,
  "internal": false,
  "auth_provider": {
    "name": "",
    "storage_engine": "",
    "meta": null
  },
  "session_provider": {
    "name": "",
    "storage_engine": "",
    "meta": null
  },
  "event_handlers": {
    "events": null
  },
  "enable_batch_request_support": false,
  "enable_ip_whitelisting": false,
  "allowed_ips": null,
  "enable_ip_blacklisting": false,
  "blacklisted_ips": null,
  "dont_set_quota_on_create": false,
  "expire_analytics_after": 0,
  "response_processors": null,
  "CORS": {
    "enable": false,
    "allowed_origins": null,
    "allowed_methods": null,
    "allowed_headers": null,
    "exposed_headers": null,
    "allow_credentials": false,
    "max_age": 0,
    "options_passthrough": false,
    "debug": false
  },
  "domain": "",
  "certificates": null,
  "do_not_track": false,
  "tags": [],
  "enable_context_vars": false,
  "config_data": {},
  "tag_headers": null,
  "global_rate_limit": {
    "rate": 0,
    "per": 0
  },
  "strip_auth_data": false,
  "enable_detailed_recording": false,
  "graphql": {
    "enabled": false,
    "execution_mode": "",
    "schema": "",
    "type_field_configurations": null,
    "playground": {
      "enabled": false,
      "path": ""
    }
  }
}

Checklist

• Make sure you are requesting to pull a topic/feature/bugfix branch (right side). If pulling from your own
  fork, don't request your master!
• Make sure you are making a pull request against the master branch (left side). Also, you should start
  your branch off our latest master.
• My change requires a change to the documentation.
  • If you've changed APIs, describe what needs to be updated in the documentation.
  • If new config option added, ensure that it can be set via ENV variable
• I have updated the documentation accordingly.
• Modules and vendor dependencies have been updated; run go mod tidy && go mod vendor
• When updating library version must provide reason/explanation for this update.
• I have added tests to cover my changes.
• All new and existing tests passed.
• Check your code additions will not fail linting checks:
  • go fmt -s
  • go vet

[Moses-oyedeji]

@anx-hnezbeda Thanks for contacting Tyk!
We apologize for the delayed response. The pull request is currently being reviewed and we will keep you updated as soon as we can

[nezhar]

PR has been updated to the current master

[PatrickTaibel]

I would like to revive this pull request. Our issue here is that every field that is parsed as URLSpec does not support regex quantifiers that are wrapped with { }. As I can see from the documentation of URL Rewriting this is a feature for wildcard patterns in Tyk. Problem is that this breaks more sophisticated regex expressions.
I guess that the current state of this PR is not really helpful for you as it would remove the wildcard path feature completely. I would suggest that we introduce a global server setting for switching between full regex mode and the mixed approach with the wildcard patterns.

[titpetric]

@PatrickTaibel we discussed having a particular match for the length regexp /{[0-9]+(,[0-9]+)}/, to keep that regex fragment as-is, and not apply matching for gorilla/mux-like /{id} patterns in such a case. The config flag makes sense for completely new behavior, but we can agree that {8} and {2,8} are valid regex patterns that should have stayed in effect (bug). No decision yet, but we have a discussion planned on this.

[andyo-tyk]

This issue was fixed in Tyk 5.5.1.