TypeScript バイナリデータ読み込み処理

バイナリデータを少しずつ数値や文字列に変換しながら読み込んでいく処理です。以前の記事の巨大ファイルの部分読み込みと組み合わせると省メモリ＆高速なデータ解析処理が実現できます。

使い方

fetchで取得したBlobを使った例です。

let blob = await (await fetch("./data.bin")).blob();
let reader = new BinaryReader(new BlobBinaryReadIF(blob), 1024);
let u32 = await reader.getU32();// integer
let f64 = await reader.getF64();// float
let u16be = await reader.getU16BE();// Big Endian
let vint = await reader.getVINT();// variable int
let str = await reader.getUTF8(8);// string UTF8

部分読み込みについては以下の記事を参考に
BinaryReadIFインターフェイスを実装してください。

実装ソースコード

// データ読み込みインターフェイス
// get関数で指定した位置のデータ取得
interface BinaryReadIF{
    get(begin : number, end : number) : Promise<ArrayBuffer>,
    size : number,
}
// Blob or File 読み込みIF
export class BlobBinaryReadIF{
    blob : Blob|File;
   
    constructor(blob : Blob|File){
        this.blob = blob;
    }
    get size() { return this.blob.size;} 
    async get(begin : number, end : number){
        return this.blob.slice(begin, end).arrayBuffer();
    }
}

// バイナリー読み込みストリーム
export class BinaryReader{
    private reader : BinaryReadIF;
    private readPtr : number;
    readonly preFetchSize : number;
    private bton : BinaryToNumber;
    private text : TextDecoder;
    private prePtr : number;
    private preBuff : ArrayBuffer;

    // preFetchSize : 先読みサイズ　0なら先読み無効　ファイルアクセスの高速化
    constructor(reader : BinaryReadIF, preFetchSize = 1024*1024){
        this.reader = reader;
        this.readPtr = 0;
        this.preFetchSize = preFetchSize;
        this.bton = new BinaryToNumber;
        this.text = new TextDecoder;
        this.prePtr = 0;
        this.preBuff = new ArrayBuffer(0);
    }

    get now(){ return this.readPtr; }
    get eof(){ return this.readPtr >= this.reader.size; } 
    get size(){ return this.reader.size; }
    
    // 8bit
    async getU8() {
        return this.bton.u8(await this.get(1));
    }
    async getI8() {
        return this.bton.i8(await this.get(1));
    }

    // 16bit
    async getU16(){
        return this.bton.u16(await this.get(2));
    }
    async getI16(){
        return this.bton.i16(await this.get(2));
    }
    async getU16BE(){
        return this.bton.u16BE(await this.get(2));
    }
    async getI16BE(){
        return this.bton.i16BE(await this.get(2));
    }

    // 24bit
    async getU24(){
        return this.bton.u24(await this.get(3));
    }
    async getI24(){
        return this.bton.i24(await this.get(3));
    }
    async getU24BE(){
        return this.bton.u24BE(await this.get(3));
    }
    async getI24BE(){
        return this.bton.i24BE(await this.get(3));
    }

    // 32bit
    async getU32(){
        return this.bton.u32(await this.get(4));
    }
    async getI32(){
        return this.bton.i32(await this.get(4));
    }
    async getU32BE(){
        return this.bton.u32BE(await this.get(4));
    }
    async getI32BE(){
        return this.bton.i32BE(await this.get(4));
    }

    // 32bit float
    async getF32(){
        return this.bton.f32(await this.get(4));
    }
    async getF32BE(){
        return this.bton.f32BE(await this.get(4));
    }

    // 64bit float
    async getF64(){
        return this.bton.f64(await this.get(8));
    }
    async getF64BE(){
        return this.bton.f64BE(await this.get(8));
    }

    // 64bit 制限あり　48bitまで
    async getU64BE(){
        let a8 = await this.get(8);
        // 最初2byte無視は無視
        return (
            ((a8[2] & 0xff)*256*256*256*256*256)
            + ((a8[3] & 0xff)*256*256*256*256)
            + ((a8[4] & 0xff)*256*256*256)
            + ((a8[5] & 0xff)*256*256)
            + ((a8[6] & 0xff)*256)
            + ((a8[7] & 0xff))
        );
    }

    // variable size integer
    async getVINT(){
        let top = await this.getU8();
        let size = 1;
        let val = 0;
        let bit = 0x80;
        let mask = 0x7f;
        for(;size<8;++size){
            if(top & bit){
                val = top & mask;
                break;
            }
            bit = bit >> 1;
            mask = mask >> 1;
        }
       
        // 7byte 49bitまで　doubleでの整数計算限界
        for(let i=1;i<size;++i){
            let v = await this.getU8();
            val = (val*256) + v;
        }
        if(size > 7){
            // throw "";
            // webmのサイズ不定など
            return NaN;
        }
        return val;
    }

    async getUTF8(len : number){
        let bin = await this.get(len);
        return this.text.decode(bin);
    }

    async getUintBE( len : number ){
        let bin = await this.get(len);
        let val = 0;
        for(let i=0;i<bin.length;++i){
            val = val*256 + bin[i];
        }
        return val;
    }
    async getFloatBE(size : number){
        if(size==4){
            return await this.getF32BE();
        }else if(size == 8){
            return await this.getF64BE();
        }else{
            throw "ReadStream::getFloat";
        }
    }
    
    skip(len : number){
        this.readPtr += len;
        if(this.readPtr > this.reader.size){ this.readPtr = this.reader.size; }
    }

    async get(len : number) : Promise<Uint8Array>{
        if( len >= this.preFetchSize){
            // 大きい場合は、直接読み込み
            let begin = this.readPtr;
            let end = this.readPtr + len;
            if(end > this.reader.size){ end = this.reader.size; }
            this.readPtr = end;
            this.prePtr = 0;
            this.preBuff = new ArrayBuffer(0);
            return new Uint8Array(await this.reader.get(begin, end));
        }
        let ptr = this.readPtr - this.prePtr;// preBuff[ptr]
        if(ptr + len > this.preBuff.byteLength){
            // 追加　残りは無視、再取得
            let begin = this.readPtr;
            let end = this.readPtr + this.preFetchSize;
            if(end > this.reader.size){ end = this.reader.size; }
            this.preBuff = await this.reader.get(begin, end);
            this.prePtr = begin;
            ptr = 0;
        }
        let begin = ptr;
        let end = begin + len;
        this.readPtr += len;
        return new Uint8Array(this.preBuff.slice(begin, end));        
    }
}


class BinaryToNumber{
    // union array
    abuf : ArrayBuffer;
    u8a : Uint8Array;
    s8a : Int8Array;
    u16a : Uint16Array;
    s16a : Int16Array;
    u32a : Uint32Array;
    s32a : Int32Array;
    f32a : Float32Array;
    f64a : Float64Array;
    constructor(){
        this.abuf = new ArrayBuffer(8);
        this.u8a = new Uint8Array(this.abuf);
        this.s8a = new Int8Array(this.abuf);
        this.u16a = new Uint16Array(this.abuf);
        this.s16a = new Int16Array(this.abuf);
        this.u32a = new Uint32Array(this.abuf);
        this.s32a = new Int32Array(this.abuf);
        this.f32a = new Float32Array(this.abuf);
        this.f64a = new Float64Array(this.abuf);
    }
    set8(bin : Uint8Array | number[], len : number){
        for(let i=0;i<len;++i){
            this.u8a[i] = bin[i];
        }
    }
    set8be(bin : Uint8Array | number[], len : number){
        for(let i=0;i<len;++i){
            this.u8a[i] = bin[len-1-i];
        }
    }

    // 8bit
    u8(bin : Uint8Array | number[]) {
        this.set8( bin, 1 );
        return this.u8a[0];
    }
    i8(bin : Uint8Array | number[]) {
        this.set8( bin, 1 );
        return this.s8a[0];
    }

    // 16bit
    u16(bin : Uint8Array | number[]){
        this.set8( bin, 2 );
        return this.u16a[0];
    }
    i16(bin : Uint8Array | number[]){
        this.set8( bin, 2 );
        return this.s16a[0];
    }
    u16BE(bin : Uint8Array | number[]){
        this.set8be( bin, 2);
        return this.u16a[0];
    }
    i16BE(bin : Uint8Array | number[]){
        this.set8be( bin, 2);
        return this.s16a[0];
    }

    // 24bit 
    u24(bin : Uint8Array | number[]){
        this.u8a[0] = bin[0];
        this.u8a[1] = bin[1];
        this.u8a[2] = bin[2];
        this.u8a[3] = 0;
        return this.u32a[0];
    }
    i24(bin : Uint8Array | number[]){
        this.u8a[0] = 0;
        this.u8a[1] = bin[0];
        this.u8a[2] = bin[1];
        this.u8a[3] = bin[2];
        return this.s32a[0]/256;
    }
    u24BE(bin : Uint8Array | number[]){
        this.u8a[0] = bin[2];
        this.u8a[1] = bin[1];
        this.u8a[2] = bin[0];
        this.u8a[3] = 0;
        return this.u32a[0];
    }
    i24BE(bin : Uint8Array | number[]){
        this.u8a[0] = 0;
        this.u8a[1] = bin[2];
        this.u8a[2] = bin[1];
        this.u8a[3] = bin[0];
        return this.s32a[0]/256;
    }

    // 32bit
    u32(bin : Uint8Array | number[]){
        this.set8( bin, 4 );
        return this.u32a[0];
    }
    u32BE(bin : Uint8Array | number[]){
        this.set8be( bin, 4 );
        return this.u32a[0];
    }
    i32(bin : Uint8Array | number[]){
        this.set8( bin, 4 );
        return this.s32a[0];
    }
    i32BE(bin : Uint8Array | number[]){
        this.set8be( bin, 4);
        return this.s32a[0];
    }

    // 32bit float
    f32(bin : Uint8Array | number[]){
        this.set8( bin, 4);
        return this.f32a[0];
    }
    f32BE(bin : Uint8Array | number[]){
        this.set8be( bin, 4);
        return this.f32a[0];
    }

    // 64bit float
    f64(bin : Uint8Array | number[]){
        this.set8( bin, 8);
        return this.f64a[0];
    }
    f64BE(bin : Uint8Array | number[]){
        this.set8be( bin, 8);
        return this.f64a[0];
    }
}