SCALE Codec
Input and Output
Understanding the I/O abstraction layer for encoding and decoding
I/O System Overview
The polkadart_scale_codec package uses an abstraction layer for input and output operations. This design allows codecs to work with different data sources and destinations without changing the encoding/decoding logic.
Input Interface
The Input mixin provides methods for reading encoded data.
Creating Input
There are multiple ways to create an Input:
import 'package:polkadart_scale_codec/polkadart_scale_codec.dart';
// From hex string
final input1 = Input.fromHex('0x2a000000');
// From bytes (List<int>)
final input2 = Input.fromBytes([42, 0, 0, 0]);
// From Uint8List
final bytes = Uint8List.fromList([42, 0, 0, 0]);
final input3 = ByteInput(bytes);
Reading Data
Single Byte
final input = Input.fromHex('0x010203');
final byte1 = input.read(); // 1
final byte2 = input.read(); // 2
final byte3 = input.read(); // 3
Multiple Bytes
final input = Input.fromHex('0x01020304050607');
final bytes = input.readBytes(4); // [1, 2, 3, 4]
// Remaining: [5, 6, 7]
Peeking (Non-Consuming)
final input = Input.fromHex('0x010203');
// Peek without consuming
final byte = input.peekByte(0); // 1
// Input position unchanged
// Peek multiple bytes
final bytes = input.peekBytes(0, 2); // [1, 2]
// Input position still unchanged
// Now consume
final consumed = input.read(); // 1
// Now position advanced
Input State
Remaining Length
final input = Input.fromHex('0x0102030405');
print(input.remainingLength); // 5
input.readBytes(2);
print(input.remainingLength); // 3
Checking for Bytes
final input = Input.fromHex('0x0102');
while (input.hasBytes()) {
final byte = input.read();
print(byte);
}
print(input.hasBytes()); // false
Reset Offset
final input = Input.fromHex('0x010203');
input.read(); // 1
input.read(); // 2
// Reset to beginning
input.resetOffset();
input.read(); // 1 again
Cloning Input
final input = Input.fromHex('0x010203');
// Clone for separate processing
final clone = input.clone();
// Original and clone are independent
input.read(); // 1
clone.read(); // 1 (separate position)
Hex Conversion
final input = Input.fromBytes([1, 2, 3, 4]);
// Get hex representation
print(input.toHex()); // 0x01020304
Accessing Buffer
final input = Input.fromHex('0x01020304');
// Get underlying buffer
final buffer = input.buffer; // Uint8List [1, 2, 3, 4]
End of Data Validation
final input = Input.fromHex('0x2a');
final value = U8Codec.codec.decode(input);
// Verify all data consumed
input.assertEndOfDataReached();
// Throws if bytes remain
// With custom message
input.assertEndOfDataReached(' Expected complete message.');
Output Interface
The Output mixin provides methods for writing encoded data.
Output Types
ByteOutput
Collects bytes into a buffer:
final output = ByteOutput();
output.write([1, 2, 3]);
output.pushByte(4);
final bytes = output.toBytes(); // Uint8List [1, 2, 3, 4]
HexOutput
Automatically converts to hex:
final output = HexOutput();
output.write([42, 0, 0, 0]);
print(output.toString()); // 0x2a000000
GeneratorOutput
For streaming output:
final output = GeneratorOutput();
output.write([1, 2]);
output.write([3, 4]);
// Stream the data
for (final byte in output.generator) {
print(byte);
}
SizeTracker
Calculates size without storing data:
final tracker = SizeTracker();
U32Codec.codec.encodeTo(42, tracker);
StrCodec.codec.encodeTo('Hello', tracker);
print(tracker.size); // Total bytes: 4 + 6 = 10
Writing Data
Single Byte
final output = ByteOutput();
output.pushByte(42);
output.pushByte(100);
print(output.toBytes()); // [42, 100]
Multiple Bytes
final output = ByteOutput();
output.write([1, 2, 3]);
output.write([4, 5, 6]);
print(output.toBytes()); // [1, 2, 3, 4, 5, 6]
Using with Codecs
Encoding Pattern
// Create output
final output = ByteOutput();
// Encode using codec
U32Codec.codec.encodeTo(42, output);
BoolCodec.codec.encodeTo(true, output);
StrCodec.codec.encodeTo('Hello', output);
// Get result
final encoded = output.toBytes();
// Or use helper
final encoded2 = U32Codec.codec.encode(42);
Decoding Pattern
// Create input from data
final input = Input.fromBytes(encodedData);
// Decode using codec
final value1 = U32Codec.codec.decode(input);
final value2 = BoolCodec.codec.decode(input);
final value3 = StrCodec.codec.decode(input);
// Verify all consumed
input.assertEndOfDataReached();
Complete Example: Custom Protocol
Here's a complete example implementing a simple protocol:
import 'package:polkadart_scale_codec/polkadart_scale_codec.dart';
class Message {
final int version;
final int messageType;
final List<int> payload;
Message(this.version, this.messageType, this.payload);
// Encode message
Uint8List encode() {
final output = ByteOutput();
// Header
U8Codec.codec.encodeTo(version, output);
U8Codec.codec.encodeTo(messageType, output);
// Payload with length prefix
CompactCodec.codec.encodeTo(payload.length, output);
output.write(payload);
return output.toBytes();
}
// Decode message
static Message decode(Uint8List data) {
final input = ByteInput(data);
// Read header
final version = U8Codec.codec.decode(input);
final messageType = U8Codec.codec.decode(input);
// Read payload
final payloadLength = CompactCodec.codec.decode(input);
final payload = input.readBytes(payloadLength);
// Verify complete
input.assertEndOfDataReached();
return Message(version, messageType, payload);
}
// Get hex representation
String toHex() {
final output = HexOutput();
U8Codec.codec.encodeTo(version, output);
U8Codec.codec.encodeTo(messageType, output);
CompactCodec.codec.encodeTo(payload.length, output);
output.write(payload);
return output.toString();
}
// Calculate encoded size
int encodedSize() {
final tracker = SizeTracker();
U8Codec.codec.encodeTo(version, tracker);
U8Codec.codec.encodeTo(messageType, tracker);
CompactCodec.codec.encodeTo(payload.length, tracker);
tracker.write(payload);
return tracker.size;
}
}
void main() {
// Create message
final msg = Message(1, 42, [1, 2, 3, 4, 5]);
print('Encoded size: ${msg.encodedSize()} bytes');
print('Hex: ${msg.toHex()}');
// Encode
final encoded = msg.encode();
// Decode
final decoded = Message.decode(encoded);
print('Version: ${decoded.version}');
print('Type: ${decoded.messageType}');
print('Payload: ${decoded.payload}');
}
Advanced Patterns
Streaming Decode
For large data, process incrementally:
void processLargeSequence(Input input) {
// Read length
final count = CompactCodec.codec.decode(input);
// Process items one at a time
for (var i = 0; i < count; i++) {
final item = U32Codec.codec.decode(input);
processItem(item);
// Item processed and can be garbage collected
}
}
void processItem(int item) {
// Process individual item
print('Processing: $item');
}
Conditional Decoding
Read structure based on runtime data:
void decodeConditional(Input input) {
final version = U8Codec.codec.decode(input);
if (version == 1) {
// Version 1 format
final data = U32Codec.codec.decode(input);
} else if (version == 2) {
// Version 2 format (different structure)
final data1 = U64Codec.codec.decode(input);
final data2 = BoolCodec.codec.decode(input);
}
}
Error Recovery
Handle partial decoding:
Result<Message, String> safeDecode(Uint8List data) {
try {
final input = ByteInput(data);
final message = Message.decode(data);
input.assertEndOfDataReached();
return Result.ok(message);
} catch (e) {
return Result.err('Decode failed: $e');
}
}
Batched Encoding
Encode multiple items efficiently:
Uint8List encodeMany(List<int> values) {
final output = ByteOutput();
// Write count
CompactCodec.codec.encodeTo(values.length, output);
// Write all values
for (final value in values) {
U32Codec.codec.encodeTo(value, output);
}
return output.toBytes();
}
Performance Tips
Pre-allocate Output
// Calculate size first
final expectedSize = codec.sizeHint(value);
// Pre-allocate buffer
final output = ByteOutput(expectedSize);
// Encode (no reallocation needed)
codec.encodeTo(value, output);
Reuse Outputs
class Encoder {
final ByteOutput _output = ByteOutput();
Uint8List encode(int value) {
_output.clear(); // Clear previous data
U32Codec.codec.encodeTo(value, _output);
return _output.toBytes();
}
}
Use Size Tracker
// Don't encode just to get size
final bytes = codec.encode(value);
final size = bytes.length; // Wasteful
// Use size tracker instead
final size = codec.sizeHint(value); // Efficient
ByteInput Details
The ByteInput class is the main implementation:
class ByteInput with Input {
final Uint8List _buffer;
int offset = 0;
ByteInput(this._buffer);
@override
int read() {
return _buffer[offset++];
}
@override
Uint8List readBytes(int length) {
final result = _buffer.sublist(offset, offset + length);
offset += length;
return Uint8List.fromList(result);
}
// ... other methods
}
Buffer Management
final buffer = Uint8List.fromList([1, 2, 3, 4]);
final input = ByteInput(buffer);
// Current position
print(input.offset); // 0
// Read and advance
input.read();
print(input.offset); // 1
// Reset
input.resetOffset();
print(input.offset); // 0
Common Patterns
Encode-Decode Round-Trip
void testRoundTrip<T>(Codec<T> codec, T value) {
// Encode
final encoded = codec.encode(value);
// Decode
final decoded = codec.decode(Input.fromBytes(encoded));
// Verify
assert(decoded == value, 'Round-trip failed');
}
Partial Decode
Map<String, dynamic> decodeHeader(Input input) {
return {
'version': U8Codec.codec.decode(input),
'type': U8Codec.codec.decode(input),
'length': CompactCodec.codec.decode(input),
};
// Input still has body data remaining
}
Multi-Format Output
void showMultiFormat(int value) {
// Bytes
final bytes = U32Codec.codec.encode(value);
print('Bytes: $bytes');
// Hex
final hexOutput = HexOutput();
U32Codec.codec.encodeTo(value, hexOutput);
print('Hex: ${hexOutput.toString()}');
// Size
final size = U32Codec.codec.sizeHint(value);
print('Size: $size bytes');
}
Best Practices
Always Verify End of Data
After decoding, verify all data was consumed:final decoded = codec.decode(input);
input.assertEndOfDataReached();
Use Hex for Debugging
HexOutput makes it easy to inspect encoded data:final output = HexOutput();
codec.encodeTo(value, output);
print('Debug: ${output.toString()}');
Handle Buffer Exhaustion
Always check remaining length:if (input.remainingLength! < 4) {
throw Exception('Not enough data for u32');
}
Clone for Lookahead
When you need to inspect data without consuming:final clone = input.clone();
final peekedValue = codec.decode(clone);
// Original input unchanged
Next Steps
- Best Practices - Learn encoding best practices
- Examples - See real-world usage examples
- substrate_metadata - Learn how SCALE integrates with metadata