Skip to content

Commit

Permalink
Microsoft.Data.SQLite: Bring back the SqliteBlob optimizations
Browse files Browse the repository at this point in the history 
This reverts commits 2fdcc8d and b416e9d.

Fixes #13987
  • Loading branch information
@bricelam
bricelam committed Feb 28, 2020
1 parent 1917350 commit 2fad00e
Show file tree
Hide file tree
Showing 3 changed files with 203 additions and 52 deletions.
10 changes: 10 additions & 0 deletions src/Microsoft.Data.Sqlite.Core/SqliteDataReader.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -544,6 +544,16 @@ public override Stream GetStream(int ordinal)
? throw new InvalidOperationException(Resources.NoData)
: _record.GetStream(ordinal);

/// <summary>
/// Retrieves data as a <see cref="TextReader" />.
/// </summary>
/// <param name="ordinal">The zero-based column ordinal.</param>
/// <returns>The returned object.</returns>
public override TextReader GetTextReader(int ordinal)
=> IsDBNull(ordinal)
? (TextReader)new StringReader(string.Empty)
: new StreamReader(GetStream(ordinal), Encoding.UTF8);

/// <summary>
/// Gets the value of the specified column.
/// </summary>
Expand Down
118 changes: 66 additions & 52 deletions src/Microsoft.Data.Sqlite.Core/SqliteDataRecord.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
using System.Diagnostics;
using System.IO;
using System.Linq;
using System.Text;
using Microsoft.Data.Sqlite.Properties;
using SQLitePCL;
using static SQLitePCL.raw;
Expand All @@ -17,6 +18,7 @@ internal class SqliteDataRecord : SqliteValueReader, IDisposable
private readonly byte[][] _blobCache;
private readonly int?[] _typeCache;
private bool _stepped;
private int? _rowidOrdinal;

public SqliteDataRecord(sqlite3_stmt stmt, bool hasRows, SqliteConnection connection)
{
Expand Down Expand Up @@ -196,26 +198,32 @@ public static Type GetFieldType(string type)

public virtual long GetBytes(int ordinal, long dataOffset, byte[] buffer, int bufferOffset, int length)
{
var blob = GetCachedBlob(ordinal);
using var stream = GetStream(ordinal);

long bytesToRead = blob.Length - dataOffset;
if (buffer != null)
if (buffer == null)
{
bytesToRead = Math.Min(bytesToRead, length);
Array.Copy(blob, dataOffset, buffer, bufferOffset, bytesToRead);
return stream.Length - dataOffset;
}

return bytesToRead;
stream.Position = dataOffset;

return stream.Read(buffer, bufferOffset, length);
}

public virtual long GetChars(int ordinal, long dataOffset, char[] buffer, int bufferOffset, int length)
{
var text = GetString(ordinal);
using var reader = new StreamReader(GetStream(ordinal), Encoding.UTF8);

int charsToRead = text.Length - (int)dataOffset;
charsToRead = Math.Min(charsToRead, length);
text.CopyTo((int)dataOffset, buffer, bufferOffset, charsToRead);
return charsToRead;
for (var position = 0; position < dataOffset; position++)
{
if (reader.Read() == -1)
{
// NB: Message is provided by the framework
throw new ArgumentOutOfRangeException(nameof(dataOffset), dataOffset, message: null);
}
}

return reader.Read(buffer, bufferOffset, length);
}

public virtual Stream GetStream(int ordinal)
Expand All @@ -229,59 +237,65 @@ public virtual Stream GetStream(int ordinal)
var blobDatabaseName = sqlite3_column_database_name(Handle, ordinal).utf8_to_string();
var blobTableName = sqlite3_column_table_name(Handle, ordinal).utf8_to_string();

var rowidOrdinal = -1;
for (var i = 0; i < FieldCount; i++)
if (!_rowidOrdinal.HasValue)
{
if (i == ordinal)
{
continue;
}
_rowidOrdinal = -1;

var databaseName = sqlite3_column_database_name(Handle, i).utf8_to_string();
if (databaseName != blobDatabaseName)
for (var i = 0; i < FieldCount; i++)
{
continue;
if (i == ordinal)
{
continue;
}

var databaseName = sqlite3_column_database_name(Handle, i).utf8_to_string();
if (databaseName != blobDatabaseName)
{
continue;
}

var tableName = sqlite3_column_table_name(Handle, i).utf8_to_string();
if (tableName != blobTableName)
{
continue;
}

var columnName = sqlite3_column_origin_name(Handle, i).utf8_to_string();
if (columnName == "rowid")
{
_rowidOrdinal = i;
break;
}

var rc = sqlite3_table_column_metadata(
_connection.Handle,
databaseName,
tableName,
columnName,
out var dataType,
out var collSeq,
out var notNull,
out var primaryKey,
out var autoInc);
SqliteException.ThrowExceptionForRC(rc, _connection.Handle);
if (string.Equals(dataType, "INTEGER", StringComparison.OrdinalIgnoreCase)
&& primaryKey != 0)
{
_rowidOrdinal = i;
break;
}
}

var tableName = sqlite3_column_table_name(Handle, i).utf8_to_string();
if (tableName != blobTableName)
{
continue;
}

var columnName = sqlite3_column_origin_name(Handle, i).utf8_to_string();
if (columnName == "rowid")
{
rowidOrdinal = i;
break;
}

var rc = sqlite3_table_column_metadata(
_connection.Handle,
databaseName,
tableName,
columnName,
out var dataType,
out var collSeq,
out var notNull,
out var primaryKey,
out var autoInc);
SqliteException.ThrowExceptionForRC(rc, _connection.Handle);
if (string.Equals(dataType, "INTEGER", StringComparison.OrdinalIgnoreCase)
&& primaryKey != 0)
{
rowidOrdinal = i;
break;
}
Debug.Assert(_rowidOrdinal.HasValue);
}

if (rowidOrdinal < 0)
if (_rowidOrdinal.Value < 0)
{
return new MemoryStream(GetCachedBlob(ordinal), false);
}

var blobColumnName = sqlite3_column_origin_name(Handle, ordinal).utf8_to_string();
var rowid = GetInt32(rowidOrdinal);
var rowid = GetInt32(_rowidOrdinal.Value);

return new SqliteBlob(_connection, blobTableName, blobColumnName, rowid, readOnly: true);
}
Expand Down
127 changes: 127 additions & 0 deletions test/Microsoft.Data.Sqlite.Tests/SqliteDataReaderTest.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -127,6 +127,27 @@ public void GetBytes_works()
}
}

[Fact]
public void GetBytes_works_streaming()
{
using (var connection = new SqliteConnection("Data Source=:memory:"))
{
connection.Open();

connection.ExecuteNonQuery("CREATE TABLE Data (Value); INSERT INTO Data VALUES (x'01020304');");

using (var reader = connection.ExecuteReader("SELECT rowid, Value FROM Data;"))
{
var hasData = reader.Read();
Assert.True(hasData);

var buffer = new byte[2];
reader.GetBytes(1, 1, buffer, 0, buffer.Length);
Assert.Equal(new byte[] { 0x02, 0x03 }, buffer);
}
}
}

[Fact]
public void GetBytes_NullBuffer()
{
Expand Down Expand Up @@ -252,6 +273,26 @@ public void GetChars_works_with_overflow()
}
}

[Fact]
public void GetChars_throws_when_dataOffset_out_of_range()
{
using (var connection = new SqliteConnection("Data Source=:memory:"))
{
connection.Open();

using (var reader = connection.ExecuteReader("SELECT 'test';"))
{
var hasData = reader.Read();
Assert.True(hasData);

var buffer = new char[1];
var ex = Assert.Throws<ArgumentOutOfRangeException>(
() => reader.GetChars(0, 5, buffer, 0, buffer.Length));
Assert.Equal("dataOffset", ex.ParamName);
}
}
}

[Fact]
public void GetChars_throws_when_closed()
{
Expand All @@ -262,6 +303,27 @@ public void GetChars_throws_when_closed()
public void GetChars_throws_when_non_query()
=> X_throws_when_non_query(r => r.GetChars(0, 0, null, 0, 0));

[Fact]
public void GetChars_works_streaming()
{
using (var connection = new SqliteConnection("Data Source=:memory:"))
{
connection.Open();

connection.ExecuteNonQuery("CREATE TABLE Data (Value); INSERT INTO Data VALUES ('test');");

using (var reader = connection.ExecuteReader("SELECT rowid, Value FROM Data;"))
{
var hasData = reader.Read();
Assert.True(hasData);

var buffer = new char[2];
reader.GetChars(1, 1, buffer, 0, buffer.Length);
Assert.Equal(new[] { 'e', 's' }, buffer);
}
}
}

[Fact]
public void GetStream_works()
{
Expand Down Expand Up @@ -388,6 +450,71 @@ public void GetStream_throws_when_closed()
public void GetStream_throws_when_non_query()
=> X_throws_when_non_query(r => r.GetStream(0));

[Fact]
public void GetTextReader_works()
{
using (var connection = new SqliteConnection("Data Source=:memory:"))
{
connection.Open();

using (var reader = connection.ExecuteReader("SELECT 'test';"))
{
var hasData = reader.Read();
Assert.True(hasData);

using (var textReader = reader.GetTextReader(0))
{
Assert.IsType<MemoryStream>(Assert.IsType<StreamReader>(textReader).BaseStream);
Assert.Equal("test", textReader.ReadToEnd());
}
}
}
}

[Fact]
public void GetTextReader_works_when_null()
{
using (var connection = new SqliteConnection("Data Source=:memory:"))
{
connection.Open();

using (var reader = connection.ExecuteReader("SELECT NULL;"))
{
var hasData = reader.Read();
Assert.True(hasData);

using (var textReader = reader.GetTextReader(0))
{
Assert.IsType<StringReader>(textReader);
Assert.Empty(textReader.ReadToEnd());
}
}
}
}

[Fact]
public void GetTextReader_works_streaming()
{
using (var connection = new SqliteConnection("Data Source=:memory:"))
{
connection.Open();

connection.ExecuteNonQuery("CREATE TABLE Data (Value); INSERT INTO Data VALUES ('test');");

using (var reader = connection.ExecuteReader("SELECT rowid, Value FROM Data;"))
{
var hasData = reader.Read();
Assert.True(hasData);

using (var textReader = reader.GetTextReader(1))
{
Assert.IsType<SqliteBlob>(Assert.IsType<StreamReader>(textReader).BaseStream);
Assert.Equal("test", textReader.ReadToEnd());
}
}
}
}

[Fact]
public void GetDateTime_works_with_text()
=> GetX_works(
Expand Down

0 comments on commit 2fad00e

Please sign in to comment.