Bifrost Document

Version 1.6.x

Bifrost ---- A heterogeneous middleware which can synchronize MySQL binlog data To Redis,MongoDB,ClickHouse...

Data Write

After each data writing, it will judge whether the file size is greater than 16MB.

If it is greater than, it will mark whether a new file needs to be created next time.

const FileMaxSize int64 = 16 * 1024 * 1024
func (This *Queue) Append(content string) error{
    return This.AppendBytes([]byte(content))
}

func (This *Queue) AppendBytes(b []byte) error{
    This.Lock()
    defer This.Unlock()
    if This.writeInfo == nil{
        This.writeInfoInit()
    }
    if This.writeInfo.pos > 0 {
        This.writeInfo.fd.Write([]byte(";"))
        This.writeInfo.pos+=1
    }
    n := Int32ToBytes(int32(len(b)))
    _,err0 := This.writeInfo.fd.Write(n)
    if err0 != nil{
        log.Fatal(err0)
    }
    This.writeInfo.fd.Write(b)
    This.writeInfo.fd.Write(n)
    This.writeInfo.pos += 8+int64(len(b))

    if This.writeInfo.pos >= FileMaxSize{
        This.writeInfo.fd.Close()
        This.writeInfo = nil
    }

    return nil
}


func (This *Queue) writeInfoInit(){
    This.maxId = This.maxId+1
    fileName := This.path+"/"+fmt.Sprint(This.maxId)+".list"
    fd0,err:=os.OpenFile(fileName,os.O_RDWR|os.O_CREATE|os.O_APPEND,0700)
    if err!=nil{
        log.Fatal("filequeue writeInfoInit err:",err)
    }
    This.fileCount++
    This.writeInfo = &FileInfo{
        fd:fd0,
        name:fileName,
        pos:0,
    }
}

源码参考

Last updated on 11 Aug 2020
Published on 11 Aug 2020
Edit on GitHub