Fixed MariaDB protocol handling issue

.circleci/config.yml

@@ -2,32 +2,32 @@ version: 2
 jobs:
   build_binaries:
     docker:
-      - image: cossacklabs/ci-py-go-themis:0.93.0
+      - image: cossacklabs/ci-py-go-themis:0.95.0
     environment:
       FILEPATH_ERROR_FLAG: /tmp/test_fail
       GO_VERSIONS: 1.19
     steps:
-        # prepare
-        - checkout
-        - run:
-            command: python3 tests/generate_random_data.py
-            environment:
-              TEST_RANDOM_DATA_FOLDER: /tmp/test_data
-        - run:
-            command: ".circleci/build_binaries.sh"
-            environment:
-              OUTPUT_FOLDER: /tmp/acra-binaries
-        - save_cache:
-            # cache only for one commit, don't cache between commits because binaries should be changed
-            # but it's okay to cache them for same commit because it will avoid re-building on re-run commands
-            key: integration_tests_preparements_.{{ .Revision }}
-            paths:
-              - /tmp/test_data
-              - /tmp/acra-binaries
+      # prepare
+      - checkout
+      - run:
+          command: python3 tests/generate_random_data.py
+          environment:
+            TEST_RANDOM_DATA_FOLDER: /tmp/test_data
+      - run:
+          command: ".circleci/build_binaries.sh"
+          environment:
+            OUTPUT_FOLDER: /tmp/acra-binaries
+      - save_cache:
+          # cache only for one commit, don't cache between commits because binaries should be changed
+          # but it's okay to cache them for same commit because it will avoid re-building on re-run commands
+          key: integration_tests_preparements_.{{ .Revision }}
+          paths:
+            - /tmp/test_data
+            - /tmp/acra-binaries
 
   general_validation:
     docker:
-      - image: cossacklabs/ci-py-go-themis:0.93.0
+      - image: cossacklabs/ci-py-go-themis:0.95.0
       - image: cossacklabs/vault-ssl:1.6.2
         command: server -dev -dev-root-token-id="root_token"
         environment:
@@ -37,9 +37,9 @@ jobs:
       - image: redis:6.0.16-alpine
       # run one more redis container with tls support on 6380 port
       - image: cossacklabs/redis-ssl:7.0-alpine
-        command: ['--tls-cert-file /tmp.ssl/redis.crt', '--tls-key-file /tmp.ssl/redis.key',
-                  '--tls-ca-cert-file /tmp.ssl/ca.crt', '--tls-auth-clients yes',
-                  '--port 0', '--tls-port 6380']
+        command: [ '--tls-cert-file /tmp.ssl/redis.crt', '--tls-key-file /tmp.ssl/redis.key',
+                   '--tls-ca-cert-file /tmp.ssl/ca.crt', '--tls-auth-clients yes',
+                   '--port 0', '--tls-port 6380' ]
     environment:
       FILEPATH_ERROR_FLAG: /tmp/test_fail
       GO_VERSIONS: 1.19
@@ -92,7 +92,7 @@ jobs:
 
   postgresql-ssl:
     docker:
-      - image: cossacklabs/ci-py-go-themis:0.93.0
+      - image: cossacklabs/ci-py-go-themis:0.95.0
       - image: cossacklabs/postgresql-ssl:11-1
         environment:
           POSTGRES_PASSWORD: test
@@ -149,7 +149,7 @@ jobs:
 
   postgresql:
     docker:
-      - image: cossacklabs/ci-py-go-themis:0.93.0
+      - image: cossacklabs/ci-py-go-themis:0.95.0
       - image: postgres:11-alpine
         environment:
           POSTGRES_PASSWORD: test
@@ -206,7 +206,7 @@ jobs:
 
   postgresql-integrations:
     docker:
-      - image: cossacklabs/ci-py-go-themis:0.93.0
+      - image: cossacklabs/ci-py-go-themis:0.95.0
       - image: nsmithuk/local-kms:3.11.2
       - image: consul:1.13
       - image: cossacklabs/postgresql-ssl:11-1
@@ -265,7 +265,7 @@ jobs:
 
   postgresql-integrations-ssl:
     docker:
-      - image: cossacklabs/ci-py-go-themis:0.93.0
+      - image: cossacklabs/ci-py-go-themis:0.95.0
       - image: cossacklabs/postgresql-ssl:11-1
         environment:
           POSTGRES_PASSWORD: test
@@ -326,7 +326,7 @@ jobs:
 
   mariadb-ssl:
     docker:
-      - image: cossacklabs/ci-py-go-themis:0.93.0
+      - image: cossacklabs/ci-py-go-themis:0.95.0
       # use the same credentials for mysql db as for postgresql (which support was added first)
       # has latest tag on 2018.03.29
       - image: cossacklabs/mariadb-ssl:10.7.1
@@ -382,7 +382,7 @@ jobs:
 
   mariadb:
     docker:
-      - image: cossacklabs/ci-py-go-themis:0.93.0
+      - image: cossacklabs/ci-py-go-themis:0.95.0
       # use the same credentials for mysql db as for postgresql (which support was added first)
       # has latest tag on 2018.03.29
       - image: mariadb:latest
@@ -414,6 +414,11 @@ jobs:
       # database in neighbor container need some time to start
       # wait for the DB to be reachable, up to 15 pings with 1 second interval
       - run: NUM_PINGS=15 DELAY=1 .circleci/check_mysql_connection.sh
+      - run:
+          command: .circleci/check_gotest.sh
+          environment:
+            - TEST_BUILD_TAGS: "integration,mysql"
+            - GO_VERSIONS: 1.19
       # testing
       # generate test data for integration tests
       - run: python3 tests/generate_random_data.py
@@ -439,7 +444,7 @@ jobs:
 
   mysql-ssl:
     docker:
-      - image: cossacklabs/ci-py-go-themis:0.94.1
+      - image: cossacklabs/ci-py-go-themis:0.95.0
       # use the same credentials for mysql db as for postgresql (which support was added first)
       # has latest tag on 2018.03.29
       - image: cossacklabs/mysql-ssl:5.7.31-1
@@ -494,7 +499,7 @@ jobs:
 
   mysql:
     docker:
-      - image: cossacklabs/ci-py-go-themis:0.93.0
+      - image: cossacklabs/ci-py-go-themis:0.95.0
       - image: mysql:5.7.25
         environment:
           MYSQL_DATABASE: test
@@ -523,6 +528,11 @@ jobs:
       # database in neighbor container need some time to start
       # wait for the DB to be reachable, up to 15 pings with 1 second interval
       - run: NUM_PINGS=15 DELAY=1 .circleci/check_mysql_connection.sh
+      - run:
+          command: .circleci/check_gotest.sh
+          environment:
+            - TEST_BUILD_TAGS: "integration,mysql"
+            - GO_VERSIONS: 1.19
       # testing
       # generate test data for integration tests
       - run: python3 tests/generate_random_data.py
@@ -548,7 +558,7 @@ jobs:
 
   build_image:
     docker:
-      - image: cossacklabs/ci-py-go-themis:0.93.0
+      - image: cossacklabs/ci-py-go-themis:0.95.0
     steps:
       # prepare
       - checkout

cmd/acra-server/common/listener.go

@@ -30,12 +30,13 @@ import (
 	"github.com/cossacklabs/acra/keystore/filesystem"
 	"github.com/cossacklabs/acra/utils"
 
-	"github.com/cossacklabs/acra/decryptor/base"
-	"github.com/cossacklabs/acra/logging"
-	"github.com/cossacklabs/acra/network"
 	"github.com/prometheus/client_golang/prometheus"
 	log "github.com/sirupsen/logrus"
 	"go.opencensus.io/trace"
+
+	"github.com/cossacklabs/acra/decryptor/base"
+	"github.com/cossacklabs/acra/logging"
+	"github.com/cossacklabs/acra/network"
 )
 
 type closer func()

cmd/acra-translator/grpc_api/api_test.go

@@ -22,6 +22,10 @@ type testKeystore struct {
 	UsedID            []byte
 }
 
+func (keystore *testKeystore) GenerateHmacKey(id []byte) error {
+	panic("implement me")
+}
+
 func (keystore *testKeystore) ListRotatedKeys() ([]keystore.KeyDescription, error) {
 	panic("implement me")
 }

decryptor/mysql/column_field.go

@@ -20,28 +20,56 @@ import (
 	"encoding/binary"
 	"errors"
 
+	log "github.com/sirupsen/logrus"
+
 	"github.com/cossacklabs/acra/decryptor/mysql/base"
 	"github.com/cossacklabs/acra/logging"
-	log "github.com/sirupsen/logrus"
 )
 
+// https://dev.mysql.com/doc/dev/mysql-server/latest/group__group__cs__column__definition__flags.html#details
+const (
+	BlobFlag           = 1 << 4
+	NoDefaultValueFlag = 1 << 12
+)
+
+// Flags represent protocol ColumnDefinitionFlags
+// https://dev.mysql.com/doc/dev/mysql-server/latest/group__group__cs__column__definition__flags.html
+type Flags uint16
+
+// ContainsFlag check if specific flag contains in flagset
+func (f *Flags) ContainsFlag(flag int) bool {
+	if f == nil {
+		return false
+	}
+	return int(*f)&flag == flag
+}
+
+// RemoveFlag remove flag from flag set
+func (f *Flags) RemoveFlag(flag int) {
+	mask := ^flag
+	new := int(*f) & mask
+	*f = Flags(new)
+}
+
 // ColumnDescription https://dev.mysql.com/doc/internals/en/com-query-response.html#packet-Protocol::ColumnDefinition41
 type ColumnDescription struct {
-	changed    bool
-	originType base.Type
+	changed                 bool
+	originType              base.Type
+	mariaDBExtendedTypeInfo bool
 	// field as byte slice
-	data         []byte
-	header       []byte
-	Schema       []byte
-	Table        []byte
-	OrgTable     []byte
-	Name         []byte
-	OrgName      []byte
-	Charset      uint16
-	ColumnLength uint32
-	Type         base.Type
-	Flag         uint16
-	Decimal      uint8
+	data             []byte
+	header           []byte
+	Schema           []byte
+	Table            []byte
+	OrgTable         []byte
+	Name             []byte
+	OrgName          []byte
+	ExtendedTypeInfo []byte
+	Charset          uint16
+	ColumnLength     uint32
+	Type             base.Type
+	Flag             Flags
+	Decimal          uint8
 
 	DefaultValueLength uint64
 	DefaultValue       []byte
@@ -104,18 +132,17 @@ func ParsePrepareStatementResponse(data []byte) (*PrepareStatementResponse, erro
 }
 
 // ParseResultField parses binary field and returns ColumnDescription
-func ParseResultField(packet *Packet) (*ColumnDescription, error) {
+func ParseResultField(packet *Packet, mariaDBExtendedTypeInfo bool) (*ColumnDescription, error) {
 	field := &ColumnDescription{}
 	field.data = packet.data
 	field.header = packet.header
-
+	field.mariaDBExtendedTypeInfo = mariaDBExtendedTypeInfo
 	var n int
 	var err error
 	//skip catalog, always def
 	pos := 0
 	n, err = base.SkipLengthEncodedString(packet.data)
 	if err != nil {
-
 		return nil, err
 	}
 	pos += n
@@ -155,6 +182,29 @@ func ParseResultField(packet *Packet) (*ColumnDescription, error) {
 	}
 	pos += n
 
+	// MariaDB from 10.2 and later add additional data to packet
+	// https://mariadb.com/kb/en/result-set-packets/#column-definition-packet
+	// which represent the extended metadata info about the Column type
+	// int<lenenc> length extended info
+	//    loop
+	//       int<1> data type: 0x00:type, 0x01: format
+	//       string<lenenc> value
+	if mariaDBExtendedTypeInfo {
+		if packet.data[pos] == 0 {
+			// skip length byte
+			pos++
+		} else {
+			num, _, _, err := base.LengthEncodedInt(packet.data[pos:])
+			if err != nil {
+				return nil, err
+			}
+			// currently we dont need to take a look on extended info, so just grab it as is
+			offset := int(num + 1)
+			field.ExtendedTypeInfo = packet.data[pos : pos+offset]
+			pos += offset
+		}
+	}
+
 	//skip 0x0C constant field
 	pos++
 
@@ -171,10 +221,14 @@ func ParseResultField(packet *Packet) (*ColumnDescription, error) {
 	pos++
 
 	//flag
-	field.Flag = binary.LittleEndian.Uint16(packet.data[pos:])
+	field.Flag = Flags(binary.LittleEndian.Uint16(packet.data[pos:]))
 	pos += 2
 
-	//decimals 1
+	//decimals 1 byte
+	// max shown decimal digits:
+	//  0x00 for integers and static strings
+	//  0x1f for dynamic strings, double, float
+	//  0x00 to 0x51 for decimals
 	field.Decimal = packet.data[pos]
 	pos++
 
@@ -230,14 +284,21 @@ func (field *ColumnDescription) Dump() []byte {
 	data = append(data, base.PutLengthEncodedString(field.Name)...)
 	data = append(data, base.PutLengthEncodedString(field.OrgName)...)
 
+	if field.mariaDBExtendedTypeInfo {
+		if len(field.ExtendedTypeInfo) > 0 {
+			data = append(data, field.ExtendedTypeInfo...)
+		} else {
+			data = append(data, 0)
+		}
+	}
 	// length of fixed-length fields
 	// https://dev.mysql.com/doc/internals/en/com-query-response.html#column-definition
 	data = append(data, 0x0c)
 
 	data = append(data, base.Uint16ToBytes(field.Charset)...)
 	data = append(data, base.Uint32ToBytes(field.ColumnLength)...)
 	data = append(data, byte(field.Type))
-	data = append(data, base.Uint16ToBytes(field.Flag)...)
+	data = append(data, base.Uint16ToBytes(uint16(field.Flag))...)
 	data = append(data, field.Decimal)
 	// filler
 	data = append(data, 0, 0)

decryptor/mysql/column_field_test.go

@@ -0,0 +1,17 @@
+package mysql
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestFieldFlag(t *testing.T) {
+	flag := Flags(4241)
+	assert.True(t, flag.ContainsFlag(BlobFlag))
+	assert.True(t, flag.ContainsFlag(NoDefaultValueFlag))
+
+	flag.RemoveFlag(BlobFlag)
+	assert.False(t, flag.ContainsFlag(BlobFlag))
+	assert.True(t, flag.ContainsFlag(NoDefaultValueFlag))
+}