Flume-WAL日志机制源码分析

Sxm&

已于 2024-06-28 18:09:28 修改

阅读量535

点赞数 10

CC 4.0 BY-SA版权

分类专栏： Flume 文章标签： flume 大数据

于 2024-03-21 11:01:51 首次发布

本文链接：https://blog.csdn.net/xianmingsu/article/details/136868133

一、WAL

WAL 的全称是 Write-Ahead Logging，中文称预写式日志，是一种数据安全写入机制，记录变更操作。就是先写日志，然后在写入磁盘，这样保证数据的安全性。WAL在关系型数据库中非常常见，Mysql中的Redo Log就是采用WAL机制。

二、Put

// org.apache.flume.channel.file.Log.java
FlumeEventPointer put(long transactionID, Event event)
      throws IOException {
   
   
    Preconditions.checkState(open, "Log is closed");
    FlumeEvent flumeEvent = new FlumeEvent(
        event.getHeaders(), event.getBody());
    //封装Put操作，WAL日志会记录四种操作，分别是Put,Take,Commit和Rollback
    //Put操作，全局写顺序ID加1
    Put put = new Put(transactionID, WriteOrderOracle.next(), flumeEvent);
    ByteBuffer buffer = TransactionEventRecord.toByteBuffer(put);
    //选择数据目录的数据文件，比如log-1
    int logFileIndex = nextLogWriter(transactionID);
    long usableSpace = logFiles.get(logFileIndex).getUsableSpace();
    long requiredSpace = minimumRequiredSpace + buffer.limit();
    if (usableSpace <= requiredSpace) {
   
   
      throw new IOException("Usable space exhausted, only " + usableSpace +
          " bytes remaining, required " + requiredSpace + " bytes");
    }
    boolean error = true;
    try {
   
   
      try {
   
   
        // Put事件写入WAL日志文件，Event也就持久化到文件了
        // logFileIndex就是数据文件ID，比如log-1文件
        // 需要注意的是，这里并不意味着数据立即被物理写入到磁盘上，数据被写到了操作系统缓冲区
        // 最终由操作系统决定何时物理写入到磁盘。
        // 如果你需要确保数据已经物理写入到磁盘，可以调用FileChannel的force()方法进行强刷
        FlumeEventPointer ptr = logFiles.get(logFileIndex).put(buffer);
        error = false;
        return ptr;
      } catch (LogFileRetryableIOException e) {
   
   
        if (!open) {
   
   
          throw e;
        }
        roll(logFileIndex, buffer);
        FlumeEventPointer ptr = logFiles.get(logFileIndex).put(buffer);
        error = false;
        return ptr;
      }
    } finally {
   
   
      if (error && open) {
   
   
        roll(logFileIndex);
      }
    }
  }

//org.apache.flume.channel.file.LogFile.Writer
synchronized FlumeEventPointer put(ByteBuffer buffer) throws IOException {
   
   
      if (encryptor != null) {
   
   
        buffer = ByteBuffer.wrap(encryptor.encrypt(buffer.array()));
      }
      //往fileChannel写入数据
      Pair<Integer, Integer> pair = write(buffer);
      return new FlumeEventPointer(pair.getLeft(), pair.getRight());
    }

write(buffer)是一个公共方法，put、take、commit、rollback操作都通过该方法进行持久化。

private Pair<Integer, Integer> write(ByteBuffer buffer)
        throws IOException {
   
   
      if (!isOpen()) {
   
   
        throw new LogFileRetryableIOException("File closed " + file);
      }
      long length = position();
      long expectedLength = length + (long) buffer.limit();
      //如果一个log写不下了，将会抛出LogFileRetryableIOException，外面捕获到这个异常会创建一个新的log并进行重写，log默认大小是1.6G
      if (expectedLength > maxFileSize) {
   
   
        throw new LogFileRetryableIOException(expectedLength + " > " +
            maxFileSize);
      }
      int offset = (int) length;
      Preconditions.checkState(offset >= 0, String.valueOf(offset));
      // OP_RECORD + size + buffer
      int recordLength = 1 + (int)