某行车记录仪解析

数据块解析步骤 #

文件头 #

⽂件名: REC0000.264 ⽂件内容解析:

⽂件头:以 mxm_v20160101_00 开头,正式的头有 magic : 0x64616568 ( head )

⾸扇区 - LBA0

偏移位置 字节数 含义
0x00 16 疑似固件版本
0x10 4 MAGIC ( head )
0x14 4 索引区偏移

可以看到索引显示2048,转到2048

索引区 #

索引区仅记录关键帧位置

偏移位置 字节数 含义
0x18 4 MAGIC (前 3 字节是 idx ,最 后⼀个字节可能是序号,表⽰ 索引表的编号,可能可以是 1 、 2 、 3…… )
0x1C 4 索引条⽬的⻓度

如图索引条目长度为9520,从起始位置偏移9520,到达索引末尾。

接下来从 0x20 偏移开始就是实际的索引条⽬

索引条⽬解析 (每条记录的⻓度是 28 字节)

偏移位置 字节数 含义
0x0 1 帧类型:
0x69 ( i ): I 帧
0x70 ( p ): P 帧
0x62 ( b ): b 帧?看着不像,根据规律这种类型的数 据⻓度较短( 10 或 44 字节)
0x1 2 ⽬前在该样本⽂件内都是同 ⼀个值: 0x2e2e ,是否可 以认为是⼀个帧头 MAGIC ?
0x3 1 帧类型,和 0x0 未知上的值⼀致
(在样本⽂件中 I 帧类型的 这 4 字节为 0x69 0x2e 0x2e 0x69 , P 帧 为0x70 0x2e 0x2e 0x70 )
0x4 4 与⾸扇区中标识的时间戳相同,可能是该录像机的初始化时间
0x8 1 未知
0x9 1 通道号, 与画⾯上的通道号 相同
0xa 2 未知
0xc 8 微秒级别的时间戳,是画⾯上的时间⽔印时间
0x14 4 帧⻓度
0x18 4 帧数据的绝对偏移偏移

数据区 #

索引条⽬的末尾有 MAGIC -> 0x6f72657a ( zero ),然后 0x04 偏移位置上的 4 字节数据 是数据区的相对偏移(表⽰当前位置到数据区的距离)

该监控品牌内数据采⽤循环覆写机制,不会清空内部数据,在旧数据上覆盖

代码实现 #

package specialfile

import (
	"GoldenEyes/public/data"
	"api2/api"
	"api2/task"
	"bufio"
	"common"
	"context"
	"encoding/binary"
	"errors"
	"fmt"
	"io"
	"os"
	"sync"
	"time"
	"vmodel"
	"vmodel/base"
	"vmodel/file"
)

const (
	maxConcurrentFiles = 4 // 最大并发解析文件数
)

type DashcamParser struct {
	dataChan           chan interface{} // 数据通道,用于发送解析后的数据
	addNormalFileCount func(addNum int64)
	addDelFileCount    func(addNum int64)
	addDoneCount       func(addNum int64)
	progressUpdate     func()
	recoveredFilePool  *sync.Pool
	relateBlockPool    *sync.Pool
	limitChan          chan struct{}
}

func NewDashcamParser(
	dataChan chan interface{},
	addNormalFileCount, addDelFileCount, addDoneCount func(addNum int64),
	recoveredFilePool, relateBlockPool *sync.Pool,
	progressUpdate func(),
) *DashcamParser {
	return &DashcamParser{
		dataChan:           dataChan,
		addNormalFileCount: addNormalFileCount,
		addDelFileCount:    addDelFileCount,
		addDoneCount:       addDoneCount,
		limitChan:          make(chan struct{}, maxConcurrentFiles),
		recoveredFilePool:  recoveredFilePool,
		relateBlockPool:    relateBlockPool,
		progressUpdate:     progressUpdate,
	}
}

func (d *DashcamParser) ParseSpecialFile(ctx context.Context) error {
	_, files, err := data.GetFiles("ext == 264", false, new(api.DataQueryParams))
	if err != nil {
		api.Log.Errorf("get file failed,err:%v", err)
		return err
	}
	var wg sync.WaitGroup
	for _, f := range files {
		d.limitChan <- struct{}{} // 获取令牌
		wg.Add(1)
		go func(fullPath string) {
			defer func() {
				wg.Done()
				<-d.limitChan
			}()
			common.Log.Infof("开始解析 %s 文件", fullPath)
			err = d.parseFile(ctx, f.FullPath)
			if err != nil {
				api.Log.Errorf("parse file failed,err:%v,filePath:%s", err, f.FullPath)
				return
			}
		}(f.FullPath)
	}
	wg.Wait()
	return nil
}

func (d *DashcamParser) parseFile(ctx context.Context, fullPath string) error {
	filePath := data.GetEffectivePath(file.MountKindGoldenEyes, fullPath)
	if !common.IsFileExist(filePath) {
		return errors.New("file not exist")
	}

	f, err := os.Open(filePath)
	if err != nil {
		return fmt.Errorf("打开输入文件失败: %v", err)
	}
	defer f.Close()

	if err := d.parseFileHeader(f); err != nil {
		return err
	}

	if err := d.extractVideo(f, fullPath); err != nil {
		return err
	}
	return nil
}

func (d *DashcamParser) parseFileHeader(file *os.File) error {
	const (
		headerMagic = 0x64616568 // "head"
		idxMagic    = 0x786469   // "idx"
		dataMagic   = 0x61746164 // "data"
	)

	// 读取初始头部
	if _, err := file.Seek(0, io.SeekStart); err != nil {
		return fmt.Errorf("定位文件头失败: %w", err)
	}

	headerBuf := make([]byte, 1024)
	ioReader := bufio.NewReader(file)
	if _, err := ioReader.Read(headerBuf); err != nil {
		return fmt.Errorf("读取文件头失败: %w", err)
	}

	// 验证文件头魔数
	headMagic := binary.LittleEndian.Uint32(headerBuf[0x10:0x14])
	//fmt.Println(hex.Dump(headerBuf[0x10:0x14]))
	if headMagic != headerMagic {
		return fmt.Errorf("无效的文件格式: 文件头不匹配")
	}

	// 读取索引表偏移
	idxTableOffset := binary.LittleEndian.Uint32(headerBuf[0x14:0x18])
	if _, err := file.Seek(int64(idxTableOffset), io.SeekStart); err != nil {
		return fmt.Errorf("定位索引表失败: %w", err)
	}

	// 读取索引表头
	idxBuf := make([]byte, 0x20)
	if _, err := io.ReadFull(file, idxBuf); err != nil {
		return fmt.Errorf("读取索引表失败: %w", err)
	}

	// 验证索引表魔数
	idxMagicValue := binary.LittleEndian.Uint32(idxBuf[0x18:0x1c]) & 0xFFFFFF
	if idxMagicValue != idxMagic {
		return fmt.Errorf("无效的文件格式: 索引表不匹配")
	}

	// 跳过索引表
	idxLength := binary.LittleEndian.Uint32(idxBuf[0x1c:0x20])

	_, err := file.Seek(int64(idxLength), io.SeekCurrent)
	if err != nil {
		return fmt.Errorf("跳过索引表失败: %w", err)
	}

	// 读取数据偏移
	offsetBuf := make([]byte, 8)
	if _, err := io.ReadFull(file, offsetBuf); err != nil {
		return fmt.Errorf("读取数据偏移失败: %w", err)
	}
	dataOffset := binary.LittleEndian.Uint32(offsetBuf[4:8])

	// 定位到数据区
	_, err = file.Seek(int64(dataOffset), io.SeekCurrent)
	if err != nil {
		return fmt.Errorf("定位数据区失败: %w", err)
	}

	// 验证数据区魔数
	dataBuf := make([]byte, 8)
	if _, err := io.ReadFull(file, dataBuf); err != nil {
		return fmt.Errorf("读取数据区头失败: %w", err)
	}

	dataMagicValue := binary.LittleEndian.Uint32(dataBuf[:4])
	if dataMagicValue != dataMagic {
		return fmt.Errorf("无效的文件格式: 数据区魔数不匹配")
	}
	return nil
}

type ChannelVideoData struct {
	RecoveredFile *file.RecoveredFile // 恢复文件的元数据
	Blocks        []*file.RelateBlock // 该通道下所有帧的块信息
	Video         *file.VideoTapeInfo
	Index         uint64
}

func (d *DashcamParser) extractVideo(f *os.File, fullPath string) error {
	const (
		headerSize    = 28
		frameMagic    = 0x2e2e
		syncWordMagic = 0x692e2e69
		bufferSize    = 1024 * 1024 * 10
	)

	// 使用 bufio.Reader 减少系统调用
	reader := bufio.NewReaderSize(f, bufferSize)
	headerBuf := make([]byte, headerSize)

	isNormal := true
	channelDataMap := make(map[int]*ChannelVideoData)

	sendChannelData := func(dataMap map[int]*ChannelVideoData) {
		for _, da := range dataMap {
			d.dataChan <- da.Blocks
			d.dataChan <- da.RecoveredFile
			d.dataChan <- da.Video
			if da.Video.Status == file.NotDeleteFile {
				d.addNormalFileCount(1)
			} else {
				d.addDelFileCount(1)
			}
		}
	}

	defer func() {
		sendChannelData(channelDataMap)
		d.addDoneCount(1)
		d.progressUpdate()
	}()

	// 手动维护文件位置(避免频繁 Seek)
	currentPos, _ := f.Seek(0, io.SeekCurrent)
	//t1 := time.Now()
	for {
		frameStartPos := currentPos
		if _, err := io.ReadFull(reader, headerBuf); err != nil {
			if err == io.EOF {
				break
			}
			return fmt.Errorf("读取帧头失败: %w", err)
		}
		currentPos += headerSize

		time1 := binary.LittleEndian.Uint32(headerBuf[0:4])
		channel := int(headerBuf[9])
		displayTime := binary.LittleEndian.Uint64(headerBuf[0x0c:0x14])
		frameLength := binary.LittleEndian.Uint32(headerBuf[0x14:0x18])

		timestamp := time.UnixMicro(int64(displayTime)).Add(-8 * time.Hour)

		// 验证帧标识
		if (time1>>8)&0xFFFF != frameMagic {
			sendChannelData(channelDataMap)
			channelDataMap = make(map[int]*ChannelVideoData)
			isNormal = false

			if _, err := f.Seek(currentPos, io.SeekStart); err != nil {
				return fmt.Errorf("同步文件位置失败: %w", err)
			}

			dOffset, err := d.searchUint32InFile(f, syncWordMagic)
			if err != nil {
				return fmt.Errorf("搜索同步字失败: %w", err)
			}
			if dOffset < 0 {
				break
			}

			if _, err := f.Seek(dOffset, io.SeekStart); err != nil {
				return fmt.Errorf("文件定位失败: %w", err)
			}

			// ✅ 重置 bufio.Reader 的缓冲区
			reader.Reset(f)
			currentPos = dOffset
			continue
		}

		if frameLength > 0 {
			discarded, err := reader.Discard(int(frameLength))
			if err != nil {
				return fmt.Errorf("跳过帧数据失败: %w", err)
			}
			currentPos += int64(discarded)
		}

		d.buildChannelVideoData(channelDataMap, channel, fullPath, isNormal, uint64(frameStartPos), uint64(headerSize+frameLength), timestamp)
	}
	//fmt.Println(fullPath,  " 耗时:", time.Since(t1).Seconds())

	return nil
}
func (d *DashcamParser) searchUint32InFile(file *os.File, target uint32) (int64, error) {
	// 获取当前文件指针位置(搜索起始位置)
	startPos, err := file.Seek(0, io.SeekCurrent)
	if err != nil {
		return -1, err
	}

	// 分块读取,每次读取 64KB
	const chunkSize = 64 * 1024
	buf := make([]byte, chunkSize)

	// 用于处理跨块边界的情况(保存上一块的最后3字节)
	overlap := make([]byte, 3)
	hasOverlap := false
	currentPos := startPos

	for {
		// 读取一块数据
		n, err := file.Read(buf)
		if n == 0 {
			break
		}
		//fmt.Println(hex.Dump(buf))
		// 检查跨块边界的情况
		if hasOverlap && n >= 3 {
			// 将上一块的最后3字节和当前块的前1字节组合成4字节
			combined := append(overlap, buf[0])
			value := binary.LittleEndian.Uint32(combined)
			if value == target {
				// 找到了!返回绝对偏移
				return currentPos - 3, nil
			}
		}

		// 在当前块中搜索
		offset := searchUint32(buf[:n], target)
		if offset != -1 {
			// 找到了!返回绝对偏移
			return currentPos + int64(offset), nil
		}

		// 保存当前块的最后3字节,用于下次跨块检查
		if n >= 3 {
			copy(overlap, buf[n-3:n])
			hasOverlap = true
		}

		currentPos += int64(n)

		// 处理读取错误
		if err == io.EOF {
			break
		}
		if err != nil {
			return -1, err
		}
	}

	// 未找到
	return -1, nil
}

// searchUint32 在字节数组中搜索指定的uint32值
func searchUint32(buf []byte, target uint32) int {
	if len(buf) < 4 {
		return -1
	}

	for i := 0; i <= len(buf)-4; i++ {
		value := binary.LittleEndian.Uint32(buf[i : i+4])
		if value == target {
			return i
		}
	}

	return -1
}

// 获取或创建指定通道的视频数据
func (d *DashcamParser) buildChannelVideoData(channelDataMap map[int]*ChannelVideoData, channel int, scanFilePath string, isNormal bool, startOffset uint64, length uint64, timestamp time.Time) {

	if cvd, exists := channelDataMap[channel]; exists {
		block := d.relateBlockPool.Get().(*file.RelateBlock)
		*block = file.RelateBlock{}
		block.Cid = api.GetCaseId()
		block.Eid = api.GetEvidenceId()
		block.Tid = api.GetTaskId()
		block.Nid = task.CreateTreeNodeNid()
		block.Pid = cvd.RecoveredFile.Nid
		block.StartOffset = startOffset
		block.Length = length
		block.Index = cvd.Index
		cvd.Blocks = append(cvd.Blocks, block)
		cvd.Index += length
		//cvd.RecoveredFile.EndTime = timestamp
		cvd.RecoveredFile.DataSize += int64(length)
		cvd.Video.EndTime = timestamp
		cvd.Video.Size = cvd.RecoveredFile.DataSize
	} else {
		fileName := fmt.Sprintf("chn%03d_%s.h264", channel+1, timestamp.Format("2006-01-02 15_04_05"))
		video := &file.VideoTapeInfo{
			DataModel: base.DataModel{
				Cid: api.GetCaseId(),
				Eid: api.GetEvidenceId(),
				Tid: api.GetTaskId(),
			},
			Name:         fileName,
			NewChannel:   channel + 1,
			Status:       file.DeleteFile,
			MountKind:    file.MountKindShardmount,
			SrcFilePath:  fileName,
			StartTime:    timestamp,
			EndTime:      timestamp,
			StartOffset:  int64(startOffset),
			EndOffset:    int64(startOffset),
			FunctionName: vmodel.QuickscanTask,
		}
		if isNormal {
			video.Status = file.NotDeleteFile
		} else {
			video.Status = file.DeleteFile
		}

		recoveredData := d.recoveredFilePool.Get().(*file.RecoveredFile)
		*recoveredData = file.RecoveredFile{}
		recoveredData.Cid = api.GetCaseId()
		recoveredData.Eid = api.GetEvidenceId()
		recoveredData.Tid = api.GetTaskId()
		recoveredData.Nid = task.CreateTreeNodeNid()
		recoveredData.Channel = int64(channel)
		recoveredData.ScanFilePath = scanFilePath
		recoveredData.FileName = fileName
		recoveredData.Type = vmodel.FileTypeVideo
		recoveredData.MountKind = file.MountKindGoldenEyes
		recoveredData.FileSystem = vmodel.FileSystemFat32
		recoveredData.DataSize = int64(length)
		// 创建新的 ChannelVideoData

		block := d.relateBlockPool.Get().(*file.RelateBlock)
		*block = file.RelateBlock{}
		block.Cid = api.GetCaseId()
		block.Eid = api.GetEvidenceId()
		block.Tid = api.GetTaskId()
		block.Nid = task.CreateTreeNodeNid()
		block.Pid = recoveredData.Nid
		block.StartOffset = startOffset
		block.Length = length

		cvd := &ChannelVideoData{
			Index:         length,
			Video:         video,
			RecoveredFile: recoveredData,
			Blocks:        make([]*file.RelateBlock, 0),
		}
		cvd.Blocks = append(cvd.Blocks, block)

		channelDataMap[channel] = cvd
	}

	return
}