/*
 * Copyright 2014-2019 Real Logic Ltd.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package io.aeron.archive;

import io.aeron.Aeron;
import io.aeron.archive.client.AeronArchive;
import io.aeron.archive.client.ArchiveException;
import io.aeron.archive.codecs.*;
import io.aeron.protocol.DataHeaderFlyweight;
import org.agrona.*;
import org.agrona.collections.ArrayUtil;
import org.agrona.concurrent.EpochClock;
import org.agrona.concurrent.UnsafeBuffer;

import java.io.File;
import java.nio.ByteBuffer;
import java.nio.MappedByteBuffer;
import java.nio.channels.FileChannel;

import static io.aeron.archive.Archive.Configuration.RECORDING_SEGMENT_POSTFIX;
import static io.aeron.archive.Archive.segmentFileName;
import static io.aeron.archive.client.AeronArchive.NULL_POSITION;
import static io.aeron.archive.client.AeronArchive.NULL_TIMESTAMP;
import static io.aeron.archive.codecs.RecordingDescriptorDecoder.*;
import static io.aeron.logbuffer.FrameDescriptor.FRAME_ALIGNMENT;
import static io.aeron.protocol.DataHeaderFlyweight.HEADER_LENGTH;
import static java.nio.ByteOrder.nativeOrder;
import static java.nio.file.StandardOpenOption.*;
import static org.agrona.BitUtil.align;

/**
 * Catalog for the archive keeps details of recorded images, past and present, and used for browsing.
 * The format is simple, allocating a fixed 1KB record for each record descriptor. This allows offset
 * based look up of a descriptor in the file. The first record contains the catalog header.
 *

 * @see RecordingDescriptorHeaderDecoder
 * @see RecordingDescriptorDecoder
 * Catalog file format:
 *

 *   0                   1                   2                   3
 *   0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
 *  +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 *  |                      Descriptor Length                        |
 *  +---------------+-----------------------------------------------+
 *  |     valid     |                  Reserved                     |
 *  +---------------+-----------------------------------------------+
 *  |                          Reserved                             |
 *  +---------------------------------------------------------------+
 *  |                          Reserved                             |
 *  +---------------------------------------------------------------+
 *  |                          Reserved                             |
 *  +---------------------------------------------------------------+
 *  |                          Reserved                             |
 *  +---------------------------------------------------------------+
 *  |                          Reserved                             |
 *  +---------------------------------------------------------------+
 *  |                          Reserved                             |
 *  +---------------------------------------------------------------+
 *  |                Recording Descriptor (< 1024)                  |
 *  |                                                              ...
 * ...                                                              |
 *  +---------------------------------------------------------------+
 *  |                          Repeats...                           |
 *  |                                                              ...
 * ...                                                              |
 *  +---------------------------------------------------------------+
 *

 */
class Catalog implements AutoCloseable
{
    @FunctionalInterface
    interface CatalogEntryProcessor
    {
        void accept(
            RecordingDescriptorHeaderEncoder headerEncoder,
            RecordingDescriptorHeaderDecoder headerDecoder,
            RecordingDescriptorEncoder descriptorEncoder,
            RecordingDescriptorDecoder descriptorDecoder);
    }

    static final int PAGE_SIZE = 4096;
    static final int NULL_RECORD_ID = Aeron.NULL_VALUE;

    static final int DESCRIPTOR_HEADER_LENGTH = RecordingDescriptorHeaderDecoder.BLOCK_LENGTH;
    static final int DEFAULT_RECORD_LENGTH = 1024;
    static final long MAX_ENTRIES = calculateMaxEntries(Integer.MAX_VALUE, DEFAULT_RECORD_LENGTH);
    static final long DEFAULT_MAX_ENTRIES = 8 * 1024;
    static final byte VALID = 1;
    static final byte INVALID = 0;

    private final RecordingDescriptorHeaderDecoder descriptorHeaderDecoder = new RecordingDescriptorHeaderDecoder();
    private final RecordingDescriptorHeaderEncoder descriptorHeaderEncoder = new RecordingDescriptorHeaderEncoder();

    private final RecordingDescriptorEncoder descriptorEncoder = new RecordingDescriptorEncoder();
    private final RecordingDescriptorDecoder descriptorDecoder = new RecordingDescriptorDecoder();

    private final MappedByteBuffer catalogByteBuffer;
    private final UnsafeBuffer catalogBuffer;
    private final UnsafeBuffer fieldAccessBuffer;

    private final int recordLength;
    private final int maxDescriptorStringsCombinedLength;
    private final int maxRecordingId;
    private final boolean forceWrites;
    private final boolean forceMetadata;
    private boolean isClosed;
    private final File archiveDir;
    private final EpochClock epochClock;
    private final FileChannel catalogChannel;
    private long nextRecordingId = 0;

    Catalog(
        final File archiveDir,
        final FileChannel archiveDirChannel,
        final int fileSyncLevel,
        final long maxNumEntries,
        final EpochClock epochClock)
    {
        this.archiveDir = archiveDir;
        this.forceWrites = fileSyncLevel > 0;
        this.forceMetadata = fileSyncLevel > 1;
        this.epochClock = epochClock;

        validateMaxEntries(maxNumEntries);

        try
        {
            final File catalogFile = new File(archiveDir, Archive.Configuration.CATALOG_FILE_NAME);
            final boolean catalogPreExists = catalogFile.exists();
            final MappedByteBuffer catalogMappedByteBuffer;
            FileChannel catalogFileChannel = null;
            final long catalogLength;

            try
            {
                catalogFileChannel = FileChannel.open(catalogFile.toPath(), CREATE, READ, WRITE, SPARSE);
                if (catalogPreExists)
                {
                    catalogLength = Math.max(catalogFileChannel.size(), calculateCatalogLength(maxNumEntries));
                }
                else
                {
                    catalogLength = calculateCatalogLength(maxNumEntries);
                }

                catalogMappedByteBuffer = catalogFileChannel.map(FileChannel.MapMode.READ_WRITE, 0, catalogLength);
            }
            catch (final Exception ex)
            {
                CloseHelper.close(catalogFileChannel);
                throw new RuntimeException(ex);
            }

            catalogChannel = catalogFileChannel;
            catalogByteBuffer = catalogMappedByteBuffer;
            catalogBuffer = new UnsafeBuffer(catalogByteBuffer);
            fieldAccessBuffer = new UnsafeBuffer(catalogByteBuffer);

            final CatalogHeaderDecoder catalogHeaderDecoder = new CatalogHeaderDecoder();
            catalogHeaderDecoder.wrap(
                catalogBuffer, 0, CatalogHeaderDecoder.BLOCK_LENGTH, CatalogHeaderDecoder.SCHEMA_VERSION);

            if (catalogPreExists)
            {
                final int version = catalogHeaderDecoder.version();
                if (SemanticVersion.major(version) != AeronArchive.Configuration.MAJOR_VERSION)
                {
                    throw new ArchiveException("invalid version " + SemanticVersion.toString(version) +
                        ", archive is " + SemanticVersion.toString(AeronArchive.Configuration.SEMANTIC_VERSION));
                }

                recordLength = catalogHeaderDecoder.entryLength();
            }
            else
            {
                forceWrites(archiveDirChannel, forceWrites, forceMetadata);
                recordLength = DEFAULT_RECORD_LENGTH;

                new CatalogHeaderEncoder()
                    .wrap(catalogBuffer, 0)
                    .entryLength(DEFAULT_RECORD_LENGTH)
                    .version(AeronArchive.Configuration.SEMANTIC_VERSION);
            }

            maxDescriptorStringsCombinedLength =
                recordLength - (DESCRIPTOR_HEADER_LENGTH + RecordingDescriptorEncoder.BLOCK_LENGTH + 12);
            maxRecordingId = (int)calculateMaxEntries(catalogLength, recordLength) - 1;

            refreshCatalog(true);
        }
        catch (final Throwable ex)
        {
            close();
            throw ex;
        }
    }

    Catalog(final File archiveDir, final EpochClock epochClock)
    {
        this.archiveDir = archiveDir;
        this.forceWrites = false;
        this.forceMetadata = false;
        this.epochClock = epochClock;
        this.catalogChannel = null;

        try
        {
            final File catalogFile = new File(archiveDir, Archive.Configuration.CATALOG_FILE_NAME);
            final MappedByteBuffer catalogMappedByteBuffer;
            final long catalogLength;

            try (FileChannel channel = FileChannel.open(catalogFile.toPath(), READ, WRITE, SPARSE))
            {
                catalogLength = channel.size();
                catalogMappedByteBuffer = channel.map(FileChannel.MapMode.READ_WRITE, 0, catalogLength);
            }
            catch (final Exception ex)
            {
                throw new RuntimeException(ex);
            }

            catalogByteBuffer = catalogMappedByteBuffer;
            catalogBuffer = new UnsafeBuffer(catalogByteBuffer);
            fieldAccessBuffer = new UnsafeBuffer(catalogByteBuffer);

            final CatalogHeaderDecoder catalogHeaderDecoder = new CatalogHeaderDecoder();
            catalogHeaderDecoder.wrap(
                catalogBuffer, 0, CatalogHeaderDecoder.BLOCK_LENGTH, CatalogHeaderDecoder.SCHEMA_VERSION);

            final int version = catalogHeaderDecoder.version();
            if (SemanticVersion.major(version) != AeronArchive.Configuration.MAJOR_VERSION)
            {
                throw new ArchiveException("invalid version " + SemanticVersion.toString(version) +
                    ", archive is " + SemanticVersion.toString(AeronArchive.Configuration.SEMANTIC_VERSION));
            }

            recordLength = catalogHeaderDecoder.entryLength();
            maxDescriptorStringsCombinedLength =
                recordLength - (DESCRIPTOR_HEADER_LENGTH + RecordingDescriptorEncoder.BLOCK_LENGTH + 12);
            maxRecordingId = (int)calculateMaxEntries(catalogLength, recordLength) - 1;

            refreshCatalog(false);
        }
        catch (final Throwable ex)
        {
            close();
            throw ex;
        }
    }

    public void close()
    {
        if (!isClosed)
        {
            isClosed = true;
            CloseHelper.close(catalogChannel);
            IoUtil.unmap(catalogByteBuffer);
        }
    }

    public int maxEntries()
    {
        return maxRecordingId + 1;
    }

    public int countEntries()
    {
        return (int)nextRecordingId;
    }

    public long addNewRecording(
        final long startPosition,
        final long startTimestamp,
        final int imageInitialTermId,
        final int segmentFileLength,
        final int termBufferLength,
        final int mtuLength,
        final int sessionId,
        final int streamId,
        final String strippedChannel,
        final String originalChannel,
        final String sourceIdentity)
    {
        if (nextRecordingId > maxRecordingId)
        {
            throw new ArchiveException("catalog is full, max recordings reached: " + maxEntries());
        }

        final int combinedStringsLen = strippedChannel.length() + sourceIdentity.length() + originalChannel.length();
        if (combinedStringsLen > maxDescriptorStringsCombinedLength)
        {
            throw new ArchiveException("combined length of channel:'" + strippedChannel +
                "' and sourceIdentity:'" + sourceIdentity +
                "' and originalChannel:'" + originalChannel +
                "' exceeds max allowed:" + maxDescriptorStringsCombinedLength);
        }

        final long recordingId = nextRecordingId++;

        catalogBuffer.wrap(catalogByteBuffer, recordingDescriptorOffset(recordingId), recordLength);
        descriptorEncoder
            .wrap(catalogBuffer, DESCRIPTOR_HEADER_LENGTH)
            .recordingId(recordingId)
            .startTimestamp(startTimestamp)
            .stopTimestamp(NULL_TIMESTAMP)
            .startPosition(startPosition)
            .stopPosition(NULL_POSITION)
            .initialTermId(imageInitialTermId)
            .segmentFileLength(segmentFileLength)
            .termBufferLength(termBufferLength)
            .mtuLength(mtuLength)
            .sessionId(sessionId)
            .streamId(streamId)
            .strippedChannel(strippedChannel)
            .originalChannel(originalChannel)
            .sourceIdentity(sourceIdentity);

        descriptorHeaderEncoder
            .wrap(catalogBuffer, 0)
            .length(descriptorEncoder.encodedLength())
            .valid(VALID);

        forceWrites(catalogChannel, forceWrites, forceMetadata);

        return recordingId;
    }

    public boolean wrapDescriptor(final long recordingId, final UnsafeBuffer buffer)
    {
        if (recordingId < 0 || recordingId > maxRecordingId)
        {
            return false;
        }

        buffer.wrap(catalogByteBuffer, recordingDescriptorOffset(recordingId), recordLength);

        return descriptorLength(buffer) > 0;
    }

    public boolean wrapAndValidateDescriptor(final long recordingId, final UnsafeBuffer buffer)
    {
        if (recordingId < 0 || recordingId > maxRecordingId)
        {
            return false;
        }

        buffer.wrap(catalogByteBuffer, recordingDescriptorOffset(recordingId), recordLength);

        return descriptorLength(buffer) > 0 && isValidDescriptor(buffer);
    }

    public boolean hasRecording(final long recordingId)
    {
        return recordingId >= 0 && recordingId < nextRecordingId &&
            fieldAccessBuffer.getByte(
                recordingDescriptorOffset(recordingId) +
                    RecordingDescriptorHeaderDecoder.validEncodingOffset()) == VALID;
    }

    public void forEach(final CatalogEntryProcessor consumer)
    {
        long recordingId = 0L;
        while (wrapDescriptor(recordingId, catalogBuffer))
        {
            descriptorHeaderDecoder.wrap(
                catalogBuffer, 0, DESCRIPTOR_HEADER_LENGTH, RecordingDescriptorHeaderDecoder.SCHEMA_VERSION);

            descriptorHeaderEncoder.wrap(catalogBuffer, 0);

            descriptorDecoder.wrap(
                catalogBuffer,
                DESCRIPTOR_HEADER_LENGTH,
                RecordingDescriptorDecoder.BLOCK_LENGTH,
                RecordingDescriptorDecoder.SCHEMA_VERSION);

            descriptorEncoder.wrap(catalogBuffer, DESCRIPTOR_HEADER_LENGTH);

            consumer.accept(descriptorHeaderEncoder, descriptorHeaderDecoder, descriptorEncoder, descriptorDecoder);
            ++recordingId;
        }
    }

    public boolean forEntry(final CatalogEntryProcessor consumer, final long recordingId)
    {
        if (wrapDescriptor(recordingId, catalogBuffer))
        {
            descriptorHeaderDecoder.wrap(
                catalogBuffer, 0, DESCRIPTOR_HEADER_LENGTH, RecordingDescriptorHeaderDecoder.SCHEMA_VERSION);

            descriptorHeaderEncoder.wrap(catalogBuffer, 0);

            descriptorDecoder.wrap(
                catalogBuffer,
                DESCRIPTOR_HEADER_LENGTH,
                RecordingDescriptorDecoder.BLOCK_LENGTH,
                RecordingDescriptorDecoder.SCHEMA_VERSION);

            descriptorEncoder.wrap(catalogBuffer, DESCRIPTOR_HEADER_LENGTH);

            consumer.accept(descriptorHeaderEncoder, descriptorHeaderDecoder, descriptorEncoder, descriptorDecoder);

            return true;
        }

        return false;
    }

    public long findLast(
        final long minRecordingId, final int sessionId, final int streamId, final byte[] channelFragment)
    {
        long recordingId = nextRecordingId;
        while (--recordingId >= minRecordingId)
        {
            catalogBuffer.wrap(catalogByteBuffer, recordingDescriptorOffset(recordingId), recordLength);

            if (isValidDescriptor(catalogBuffer))
            {
                descriptorDecoder.wrap(
                    catalogBuffer,
                    DESCRIPTOR_HEADER_LENGTH,
                    RecordingDescriptorDecoder.BLOCK_LENGTH,
                    RecordingDescriptorDecoder.SCHEMA_VERSION);

                if (sessionId == descriptorDecoder.sessionId() &&
                    streamId == descriptorDecoder.streamId() &&
                    originalChannelContains(descriptorDecoder, channelFragment))
                {
                    return recordingId;
                }
            }
        }

        return NULL_RECORD_ID;
    }

    //
    // Methods for access specific record fields by recordingId.
    // Note: These methods are thread safe.
    /////////////////////////////////////////////////////////////

    public static boolean originalChannelContains(
        final RecordingDescriptorDecoder descriptorDecoder, final byte[] channelFragment)
    {
        final int fragmentLength = channelFragment.length;
        if (fragmentLength == 0)
        {
            return true;
        }

        final int limit = descriptorDecoder.limit();
        final int strippedChannelLength = descriptorDecoder.strippedChannelLength();
        final int originalChannelOffset = limit +
            RecordingDescriptorDecoder.strippedChannelHeaderLength() + strippedChannelLength;

        descriptorDecoder.limit(originalChannelOffset);
        final int channelLength = descriptorDecoder.originalChannelLength();
        descriptorDecoder.limit(limit);

        final DirectBuffer buffer = descriptorDecoder.buffer();
        int offset = descriptorDecoder.offset() + descriptorDecoder.sbeBlockLength() +
            RecordingDescriptorDecoder.strippedChannelHeaderLength() + strippedChannelLength +
            RecordingDescriptorDecoder.originalChannelHeaderLength();

        nextChar:
        for (int end = offset + (channelLength - fragmentLength); offset <= end; offset++)
        {
            for (int i = 0; i < fragmentLength; i++)
            {
                if (buffer.getByte(offset + i) != channelFragment[i])
                {
                    continue nextChar;
                }
            }

            return true;
        }

        return false;
    }

    public void recordingStopped(final long recordingId, final long position, final long timestamp)
    {
        final int offset = recordingDescriptorOffset(recordingId) + RecordingDescriptorHeaderDecoder.BLOCK_LENGTH;
        final long stopPosition = nativeOrder() == BYTE_ORDER ? position : Long.reverseBytes(position);

        fieldAccessBuffer.putLong(offset + stopTimestampEncodingOffset(), timestamp, BYTE_ORDER);
        fieldAccessBuffer.putLongVolatile(offset + stopPositionEncodingOffset(), stopPosition);

        forceWrites(catalogChannel, forceWrites, forceMetadata);
    }

    public void recordingStopped(final long recordingId, final long position)
    {
        final int offset = recordingDescriptorOffset(recordingId) + RecordingDescriptorHeaderDecoder.BLOCK_LENGTH;
        final long stopPosition = nativeOrder() == BYTE_ORDER ? position : Long.reverseBytes(position);
        fieldAccessBuffer.putLongVolatile(offset + stopPositionEncodingOffset(), stopPosition);

        forceWrites(catalogChannel, forceWrites, forceMetadata);
    }

    public void extendRecording(final long recordingId, final long controlSessionId, final long correlationId)
    {
        final int offset = recordingDescriptorOffset(recordingId) + RecordingDescriptorHeaderDecoder.BLOCK_LENGTH;

        final long stopPosition = nativeOrder() == BYTE_ORDER ? NULL_POSITION : Long.reverseBytes(NULL_POSITION);

        fieldAccessBuffer.putLong(offset + controlSessionIdEncodingOffset(), controlSessionId);
        fieldAccessBuffer.putLong(offset + correlationIdEncodingOffset(), correlationId);
        fieldAccessBuffer.putLong(offset + stopTimestampEncodingOffset(), NULL_TIMESTAMP);
        fieldAccessBuffer.putLongVolatile(offset + stopPositionEncodingOffset(), stopPosition);

        forceWrites(catalogChannel, forceWrites, forceMetadata);
    }

    public long stopPosition(final long recordingId)
    {
        final int offset = recordingDescriptorOffset(recordingId) +
            RecordingDescriptorHeaderDecoder.BLOCK_LENGTH +
            stopPositionEncodingOffset();

        final long stopPosition = fieldAccessBuffer.getLongVolatile(offset);

        return nativeOrder() == BYTE_ORDER ? stopPosition : Long.reverseBytes(stopPosition);
    }

    public RecordingSummary recordingSummary(final long recordingId, final RecordingSummary summary)
    {
        final int offset = recordingDescriptorOffset(recordingId) + RecordingDescriptorHeaderDecoder.BLOCK_LENGTH;

        summary.recordingId = recordingId;
        summary.startPosition = fieldAccessBuffer.getLong(offset + startPositionEncodingOffset(), BYTE_ORDER);
        summary.stopPosition = fieldAccessBuffer.getLong(offset + stopPositionEncodingOffset(), BYTE_ORDER);
        summary.initialTermId = fieldAccessBuffer.getInt(offset + initialTermIdEncodingOffset(), BYTE_ORDER);
        summary.segmentFileLength = fieldAccessBuffer.getInt(offset + segmentFileLengthEncodingOffset(), BYTE_ORDER);
        summary.termBufferLength = fieldAccessBuffer.getInt(offset + termBufferLengthEncodingOffset(), BYTE_ORDER);
        summary.mtuLength = fieldAccessBuffer.getInt(offset + mtuLengthEncodingOffset(), BYTE_ORDER);
        summary.streamId = fieldAccessBuffer.getInt(offset + streamIdEncodingOffset(), BYTE_ORDER);
        summary.sessionId = fieldAccessBuffer.getInt(offset + sessionIdEncodingOffset(), BYTE_ORDER);

        return summary;
    }

    public static int descriptorLength(final UnsafeBuffer descriptorBuffer)
    {
        return descriptorBuffer.getInt(RecordingDescriptorHeaderDecoder.lengthEncodingOffset(), BYTE_ORDER);
    }

    public static boolean isValidDescriptor(final UnsafeBuffer descriptorBuffer)
    {
        return descriptorBuffer.getByte(RecordingDescriptorHeaderDecoder.validEncodingOffset()) == VALID;
    }

    public static long calculateCatalogLength(final long maxEntries)
    {
        return Math.min((maxEntries * DEFAULT_RECORD_LENGTH) + DEFAULT_RECORD_LENGTH, Integer.MAX_VALUE);
    }

    public static long calculateMaxEntries(final long catalogLength, final long recordLength)
    {
        if (Integer.MAX_VALUE == catalogLength)
        {
            return (Integer.MAX_VALUE - (recordLength - 1)) / recordLength;
        }

        return (catalogLength / recordLength) - 1;
    }

    int recordingDescriptorOffset(final long recordingId)
    {
        return (int)(recordingId * recordLength) + recordLength;
    }

    public static void validateMaxEntries(final long maxEntries)
    {
        if (maxEntries < 1 || maxEntries > MAX_ENTRIES)
        {
            throw new ArchiveException(
                "Catalog max entries must be between 1 and " + MAX_ENTRIES + ": maxEntries=" + maxEntries);
        }
    }

    public static long recoverStopOffset(final File segmentFile, final int segmentFileLength)
    {
        long lastFragmentOffset = 0;
        try (FileChannel segment = FileChannel.open(segmentFile.toPath(), READ))
        {
            final ByteBuffer buffer = ByteBuffer.allocateDirect(HEADER_LENGTH);
            buffer.order(BYTE_ORDER);

            long nextFragmentOffset = 0;
            do
            {
                buffer.clear();
                if (HEADER_LENGTH != segment.read(buffer, nextFragmentOffset))
                {
                    throw new ArchiveException("unexpected read failure from file: " +
                        segmentFile.getAbsolutePath() + " at position:" + nextFragmentOffset);
                }

                final int frameLength = buffer.getInt(DataHeaderFlyweight.FRAME_LENGTH_FIELD_OFFSET);
                if (frameLength == 0)
                {
                    break;
                }

                lastFragmentOffset = nextFragmentOffset;
                nextFragmentOffset += align(frameLength, FRAME_ALIGNMENT);
            }
            while (nextFragmentOffset != segmentFileLength);

            if ((nextFragmentOffset / PAGE_SIZE) == (lastFragmentOffset / PAGE_SIZE))
            {
                // if last fragment does not straddle page boundaries we need not drop it
                lastFragmentOffset = nextFragmentOffset;
            }
        }
        catch (final Exception ex)
        {
            LangUtil.rethrowUnchecked(ex);
        }

        return lastFragmentOffset;
    }

    /**
     * On catalog load we verify entries are in coherent state and attempt to recover entries data where untimely
     * termination of recording has resulted in an unaccounted for stopPosition/stopTimestamp. This operation may be
     * expensive for large catalogs.
     *
     * @param fixOnRefresh set if the catalog should have its entries fixed.
     */
    private void refreshCatalog(final boolean fixOnRefresh)
    {
        if (fixOnRefresh)
        {
            forEach(this::refreshAndFixDescriptor);
        }
        else
        {
            forEach(((headerEncoder, headerDecoder, descriptorEncoder, descriptorDecoder) -> nextRecordingId++));
        }
    }

    private void refreshAndFixDescriptor(
        @SuppressWarnings("unused") final RecordingDescriptorHeaderEncoder unused,
        final RecordingDescriptorHeaderDecoder headerDecoder,
        final RecordingDescriptorEncoder encoder,
        final RecordingDescriptorDecoder decoder)
    {
        final long recordingId = decoder.recordingId();
        if (headerDecoder.valid() == VALID && decoder.stopPosition() == NULL_POSITION)
        {
            final String prefix = recordingId + "-";
            String[] segmentFiles = archiveDir.list(
                (dir, name) -> name.startsWith(prefix) && name.endsWith(RECORDING_SEGMENT_POSTFIX));

            if (null == segmentFiles)
            {
                segmentFiles = ArrayUtil.EMPTY_STRING_ARRAY;
            }

            int maxSegmentIndex = -1;
            for (final String filename : segmentFiles)
            {
                final int length = filename.length();
                final int offset = prefix.length();
                final int remaining = length - offset - RECORDING_SEGMENT_POSTFIX.length();

                if (remaining > 0)
                {
                    try
                    {
                        maxSegmentIndex = Math.max(
                            AsciiEncoding.parseIntAscii(filename, offset, remaining),
                            maxSegmentIndex);
                    }
                    catch (final Exception ignore)
                    {
                    }
                }
            }

            if (maxSegmentIndex < 0)
            {
                encoder.stopPosition(decoder.startPosition());
            }
            else
            {
                final File maxSegmentFile = new File(archiveDir, segmentFileName(recordingId, maxSegmentIndex));
                final int segmentFileLength = decoder.segmentFileLength();
                final long stopOffset = recoverStopOffset(maxSegmentFile, segmentFileLength);
                final int termBufferLength = decoder.termBufferLength();
                final long startPosition = decoder.startPosition();
                final long recordingLength =
                    (startPosition & (termBufferLength - 1)) + (maxSegmentIndex * (long)segmentFileLength) + stopOffset;
                encoder.stopPosition(startPosition + recordingLength);
            }

            encoder.stopTimestamp(epochClock.time());
        }

        nextRecordingId = recordingId + 1;
    }

    private void forceWrites(final FileChannel channel, final boolean forceWrites, final boolean forceMetadata)
    {
        if (null != channel && forceWrites)
        {
            try
            {
                channel.force(forceMetadata);
            }
            catch (final Exception ex)
            {
                LangUtil.rethrowUnchecked(ex);
            }
        }
    }
}


/*
 * Copyright 2014-2019 Real Logic Ltd.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package io.aeron.logbuffer;

import io.aeron.DirectBufferVector;
import io.aeron.ReservedValueSupplier;
import io.aeron.exceptions.AeronException;
import org.agrona.DirectBuffer;
import org.agrona.UnsafeAccess;
import org.agrona.concurrent.UnsafeBuffer;

import static io.aeron.logbuffer.FrameDescriptor.BEGIN_FRAG_FLAG;
import static io.aeron.logbuffer.FrameDescriptor.END_FRAG_FLAG;
import static io.aeron.logbuffer.FrameDescriptor.FRAME_ALIGNMENT;
import static io.aeron.logbuffer.FrameDescriptor.PADDING_FRAME_TYPE;
import static io.aeron.logbuffer.FrameDescriptor.frameFlags;
import static io.aeron.logbuffer.FrameDescriptor.frameLengthOrdered;
import static io.aeron.logbuffer.FrameDescriptor.frameType;
import static io.aeron.logbuffer.LogBufferDescriptor.TERM_TAIL_COUNTERS_OFFSET;
import static io.aeron.logbuffer.LogBufferDescriptor.termId;
import static io.aeron.protocol.DataHeaderFlyweight.*;
import static org.agrona.BitUtil.SIZE_OF_LONG;
import static org.agrona.BitUtil.align;
import static java.nio.ByteOrder.LITTLE_ENDIAN;

/**
 * Term buffer appender which supports many producers concurrently writing an append-only log.
 *

 * Note: This class is threadsafe.
 *

 * Messages are appended to a term using a framing protocol as described in {@link FrameDescriptor}.
 *

 * A default message header is applied to each message with the fields filled in for fragment flags, type, term number,
 * as appropriate.
 *

 * A message of type {@link FrameDescriptor#PADDING_FRAME_TYPE} is appended at the end of the buffer if claimed
 * space is not sufficiently large to accommodate the message about to be written.
 */
public final class TermAppender
{
    /**
     * The append operation failed because it was past the end of the buffer.
     */
    public static final int FAILED = -2;

    private final long tailAddressOffset;
    private final UnsafeBuffer termBuffer;

    /**
     * Construct a view over a term buffer and state buffer for appending frames.
     *
     * @param termBuffer     for where messages are stored.
     * @param metaDataBuffer for where the state of writers is stored manage concurrency.
     * @param partitionIndex for this will be the active appender.
     */
    public TermAppender(final UnsafeBuffer termBuffer, final UnsafeBuffer metaDataBuffer, final int partitionIndex)
    {
        final int tailCounterOffset = TERM_TAIL_COUNTERS_OFFSET + (partitionIndex * SIZE_OF_LONG);
        metaDataBuffer.boundsCheck(tailCounterOffset, SIZE_OF_LONG);

        this.termBuffer = termBuffer;
        tailAddressOffset = metaDataBuffer.addressOffset() + tailCounterOffset;
    }

    /**
     * Get the raw current tail value in a volatile memory ordering fashion.
     *
     * @return the current tail value.
     */
    public long rawTailVolatile()
    {
        return UnsafeAccess.UNSAFE.getLongVolatile(null, tailAddressOffset);
    }

    /**
     * Claim length of a the term buffer for writing in the message with zero copy semantics.
     *
     * @param header       for writing the default header.
     * @param length       of the message to be written.
     * @param bufferClaim  to be updated with the claimed region.
     * @param activeTermId used for flow control.
     * @return the resulting offset of the term after the append on success otherwise {@link #FAILED}.
     */
    public int claim(
        final HeaderWriter header,
        final int length,
        final BufferClaim bufferClaim,
        final int activeTermId)
    {
        final int frameLength = length + HEADER_LENGTH;
        final int alignedLength = align(frameLength, FRAME_ALIGNMENT);
        final UnsafeBuffer termBuffer = this.termBuffer;
        final int termLength = termBuffer.capacity();

        final long rawTail = getAndAddRawTail(alignedLength);
        final int termId = termId(rawTail);
        final long termOffset = rawTail & 0xFFFF_FFFFL;

        checkTerm(activeTermId, termId);

        long resultingOffset = termOffset + alignedLength;
        if (resultingOffset > termLength)
        {
            resultingOffset = handleEndOfLogCondition(termBuffer, termOffset, header, termLength, termId);
        }
        else
        {
            final int frameOffset = (int)termOffset;
            header.write(termBuffer, frameOffset, frameLength, termId);
            bufferClaim.wrap(termBuffer, frameOffset, frameLength);
        }

        return (int)resultingOffset;
    }

    /**
     * Append an unfragmented message to the the term buffer.
     *
     * @param header                for writing the default header.
     * @param bufferOne             containing the first part of the message.
     * @param offsetOne             at which the first part of the message begins.
     * @param lengthOne             of the first part of the message.
     * @param bufferTwo             containing the second part of the message.
     * @param offsetTwo             at which the second part of the message begins.
     * @param lengthTwo             of the second part of the message.
     * @param reservedValueSupplier {@link ReservedValueSupplier} for the frame.
     * @param activeTermId          used for flow control.
     * @return the resulting offset of the term after the append on success otherwise {@link #FAILED}
     */
    public int appendUnfragmentedMessage(
        final HeaderWriter header,
        final DirectBuffer bufferOne,
        final int offsetOne,
        final int lengthOne,
        final DirectBuffer bufferTwo,
        final int offsetTwo,
        final int lengthTwo,
        final ReservedValueSupplier reservedValueSupplier,
        final int activeTermId)
    {
        final int frameLength = lengthOne + lengthTwo + HEADER_LENGTH;
        final int alignedLength = align(frameLength, FRAME_ALIGNMENT);
        final UnsafeBuffer termBuffer = this.termBuffer;
        final int termLength = termBuffer.capacity();

        final long rawTail = getAndAddRawTail(alignedLength);
        final int termId = termId(rawTail);
        final long termOffset = rawTail & 0xFFFF_FFFFL;

        checkTerm(activeTermId, termId);

        long resultingOffset = termOffset + alignedLength;
        if (resultingOffset > termLength)
        {
            resultingOffset = handleEndOfLogCondition(termBuffer, termOffset, header, termLength, termId);
        }
        else
        {
            final int frameOffset = (int)termOffset;
            header.write(termBuffer, frameOffset, frameLength, termId);
            termBuffer.putBytes(frameOffset + HEADER_LENGTH, bufferOne, offsetOne, lengthOne);
            termBuffer.putBytes(frameOffset + HEADER_LENGTH + lengthOne, bufferTwo, offsetTwo, lengthTwo);

            if (null != reservedValueSupplier)
            {
                final long reservedValue = reservedValueSupplier.get(termBuffer, frameOffset, frameLength);
                termBuffer.putLong(frameOffset + RESERVED_VALUE_OFFSET, reservedValue, LITTLE_ENDIAN);
            }

            frameLengthOrdered(termBuffer, frameOffset, frameLength);
        }

        return (int)resultingOffset;
    }

    /**
     * Append an unfragmented message to the the term buffer.
     *
     * @param header                for writing the default header.
     * @param buffer                containing the message.
     * @param offset                at which the message begins.
     * @param length                of the message in the source buffer.
     * @param reservedValueSupplier {@link ReservedValueSupplier} for the frame.
     * @param activeTermId          used for flow control.
     * @return the resulting offset of the term after the append on success otherwise {@link #FAILED}
     */
    public int appendUnfragmentedMessage(
        final HeaderWriter header,
        final DirectBuffer buffer,
        final int offset,
        final int length,
        final ReservedValueSupplier reservedValueSupplier,
        final int activeTermId)
    {
        final int frameLength = length + HEADER_LENGTH;
        final int alignedLength = align(frameLength, FRAME_ALIGNMENT);
        final UnsafeBuffer termBuffer = this.termBuffer;
        final int termLength = termBuffer.capacity();

        final long rawTail = getAndAddRawTail(alignedLength);
        final int termId = termId(rawTail);
        final long termOffset = rawTail & 0xFFFF_FFFFL;

        checkTerm(activeTermId, termId);

        long resultingOffset = termOffset + alignedLength;
        if (resultingOffset > termLength)
        {
            resultingOffset = handleEndOfLogCondition(termBuffer, termOffset, header, termLength, termId);
        }
        else
        {
            final int frameOffset = (int)termOffset;
            header.write(termBuffer, frameOffset, frameLength, termId);
            termBuffer.putBytes(frameOffset + HEADER_LENGTH, buffer, offset, length);

            if (null != reservedValueSupplier)
            {
                final long reservedValue = reservedValueSupplier.get(termBuffer, frameOffset, frameLength);
                termBuffer.putLong(frameOffset + RESERVED_VALUE_OFFSET, reservedValue, LITTLE_ENDIAN);
            }

            frameLengthOrdered(termBuffer, frameOffset, frameLength);
        }

        return (int)resultingOffset;
    }

    /**
     * Append an unfragmented message to the the term buffer as a gathering of vectors.
     *
     * @param header                for writing the default header.
     * @param vectors               to the buffers.
     * @param length                of the message as a sum of the vectors.
     * @param reservedValueSupplier {@link ReservedValueSupplier} for the frame.
     * @param activeTermId          used for flow control.
     * @return the resulting offset of the term after the append on success otherwise {@link #FAILED}.
     */
    public int appendUnfragmentedMessage(
        final HeaderWriter header,
        final DirectBufferVector[] vectors,
        final int length,
        final ReservedValueSupplier reservedValueSupplier,
        final int activeTermId)
    {
        final int frameLength = length + HEADER_LENGTH;
        final int alignedLength = align(frameLength, FRAME_ALIGNMENT);
        final UnsafeBuffer termBuffer = this.termBuffer;
        final int termLength = termBuffer.capacity();

        final long rawTail = getAndAddRawTail(alignedLength);
        final int termId = termId(rawTail);
        final long termOffset = rawTail & 0xFFFF_FFFFL;

        checkTerm(activeTermId, termId);

        long resultingOffset = termOffset + alignedLength;
        if (resultingOffset > termLength)
        {
            resultingOffset = handleEndOfLogCondition(termBuffer, termOffset, header, termLength, termId);
        }
        else
        {
            final int frameOffset = (int)termOffset;
            header.write(termBuffer, frameOffset, frameLength, termId);

            int offset = frameOffset + HEADER_LENGTH;
            for (final DirectBufferVector vector : vectors)
            {
                termBuffer.putBytes(offset, vector.buffer, vector.offset, vector.length);
                offset += vector.length;
            }

            if (null != reservedValueSupplier)
            {
                final long reservedValue = reservedValueSupplier.get(termBuffer, frameOffset, frameLength);
                termBuffer.putLong(frameOffset + RESERVED_VALUE_OFFSET, reservedValue, LITTLE_ENDIAN);
            }

            frameLengthOrdered(termBuffer, frameOffset, frameLength);
        }

        return (int)resultingOffset;
    }

    /**
     * Append a fragmented message to the the term buffer.
     * The message will be split up into fragments of MTU length minus header.
     *
     * @param header                for writing the default header.
     * @param buffer                containing the message.
     * @param offset                at which the message begins.
     * @param length                of the message in the source buffer.
     * @param maxPayloadLength      that the message will be fragmented into.
     * @param reservedValueSupplier {@link ReservedValueSupplier} for the frame.
     * @param activeTermId          used for flow control.
     * @return the resulting offset of the term after the append on success otherwise  {@link #FAILED}.
     */
    public int appendFragmentedMessage(
        final HeaderWriter header,
        final DirectBuffer buffer,
        final int offset,
        final int length,
        final int maxPayloadLength,
        final ReservedValueSupplier reservedValueSupplier,
        final int activeTermId)
    {
        final int numMaxPayloads = length / maxPayloadLength;
        final int remainingPayload = length % maxPayloadLength;
        final int lastFrameLength = remainingPayload > 0 ? align(remainingPayload + HEADER_LENGTH, FRAME_ALIGNMENT) : 0;
        final int requiredLength = (numMaxPayloads * (maxPayloadLength + HEADER_LENGTH)) + lastFrameLength;
        final UnsafeBuffer termBuffer = this.termBuffer;
        final int termLength = termBuffer.capacity();

        final long rawTail = getAndAddRawTail(requiredLength);
        final int termId = termId(rawTail);
        final long termOffset = rawTail & 0xFFFF_FFFFL;

        checkTerm(activeTermId, termId);

        long resultingOffset = termOffset + requiredLength;
        if (resultingOffset > termLength)
        {
            resultingOffset = handleEndOfLogCondition(termBuffer, termOffset, header, termLength, termId);
        }
        else
        {
            int frameOffset = (int)termOffset;
            byte flags = BEGIN_FRAG_FLAG;
            int remaining = length;

            do
            {
                final int bytesToWrite = Math.min(remaining, maxPayloadLength);
                final int frameLength = bytesToWrite + HEADER_LENGTH;
                final int alignedLength = align(frameLength, FRAME_ALIGNMENT);

                header.write(termBuffer, frameOffset, frameLength, termId);
                termBuffer.putBytes(
                    frameOffset + HEADER_LENGTH,
                    buffer,
                    offset + (length - remaining),
                    bytesToWrite);

                if (remaining <= maxPayloadLength)
                {
                    flags |= END_FRAG_FLAG;
                }

                frameFlags(termBuffer, frameOffset, flags);

                if (null != reservedValueSupplier)
                {
                    final long reservedValue = reservedValueSupplier.get(termBuffer, frameOffset, frameLength);
                    termBuffer.putLong(frameOffset + RESERVED_VALUE_OFFSET, reservedValue, LITTLE_ENDIAN);
                }

                frameLengthOrdered(termBuffer, frameOffset, frameLength);

                flags = 0;
                frameOffset += alignedLength;
                remaining -= bytesToWrite;
            }
            while (remaining > 0);
        }

        return (int)resultingOffset;
    }

    /**
     * Append a fragmented message to the the term buffer.
     * The message will be split up into fragments of MTU length minus header.
     *
     * @param header                for writing the default header.
     * @param bufferOne             containing the first part of the message.
     * @param offsetOne             at which the first part of the message begins.
     * @param lengthOne             of the first part of the message.
     * @param bufferTwo             containing the second part of the message.
     * @param offsetTwo             at which the second part of the message begins.
     * @param lengthTwo             of the second part of the message.
     * @param maxPayloadLength      that the message will be fragmented into.
     * @param reservedValueSupplier {@link ReservedValueSupplier} for the frame.
     * @param activeTermId          used for flow control.
     * @return the resulting offset of the term after the append on success otherwise  {@link #FAILED}.
     */
    public int appendFragmentedMessage(
        final HeaderWriter header,
        final DirectBuffer bufferOne,
        final int offsetOne,
        final int lengthOne,
        final DirectBuffer bufferTwo,
        final int offsetTwo,
        final int lengthTwo,
        final int maxPayloadLength,
        final ReservedValueSupplier reservedValueSupplier,
        final int activeTermId)
    {
        final int length = lengthOne + lengthTwo;
        final int numMaxPayloads = length / maxPayloadLength;
        final int remainingPayload = length % maxPayloadLength;
        final int lastFrameLength = remainingPayload > 0 ? align(remainingPayload + HEADER_LENGTH, FRAME_ALIGNMENT) : 0;
        final int requiredLength = (numMaxPayloads * (maxPayloadLength + HEADER_LENGTH)) + lastFrameLength;
        final UnsafeBuffer termBuffer = this.termBuffer;
        final int termLength = termBuffer.capacity();

        final long rawTail = getAndAddRawTail(requiredLength);
        final int termId = termId(rawTail);
        final long termOffset = rawTail & 0xFFFF_FFFFL;

        checkTerm(activeTermId, termId);

        long resultingOffset = termOffset + requiredLength;
        if (resultingOffset > termLength)
        {
            resultingOffset = handleEndOfLogCondition(termBuffer, termOffset, header, termLength, termId);
        }
        else
        {
            int frameOffset = (int)termOffset;
            byte flags = BEGIN_FRAG_FLAG;
            int remaining = length;
            int positionOne = 0;
            int positionTwo = 0;

            do
            {
                final int bytesToWrite = Math.min(remaining, maxPayloadLength);
                final int frameLength = bytesToWrite + HEADER_LENGTH;
                final int alignedLength = align(frameLength, FRAME_ALIGNMENT);

                header.write(termBuffer, frameOffset, frameLength, termId);

                int bytesWritten = 0;
                int payloadOffset = frameOffset + HEADER_LENGTH;
                do
                {
                    final int remainingOne = lengthOne - positionOne;
                    if (remainingOne > 0)
                    {
                        final int numBytes = Math.min(bytesToWrite - bytesWritten, remainingOne);
                        termBuffer.putBytes(payloadOffset, bufferOne, offsetOne + positionOne, numBytes);

                        bytesWritten += numBytes;
                        payloadOffset += numBytes;
                        positionOne += numBytes;
                    }
                    else
                    {
                        final int numBytes = Math.min(bytesToWrite - bytesWritten, lengthTwo - positionTwo);
                        termBuffer.putBytes(payloadOffset, bufferTwo, offsetTwo + positionTwo, numBytes);

                        bytesWritten += numBytes;
                        payloadOffset += numBytes;
                        positionTwo += numBytes;
                    }
                }
                while (bytesWritten < bytesToWrite);

                if (remaining <= maxPayloadLength)
                {
                    flags |= END_FRAG_FLAG;
                }

                frameFlags(termBuffer, frameOffset, flags);

                if (null != reservedValueSupplier)
                {
                    final long reservedValue = reservedValueSupplier.get(termBuffer, frameOffset, frameLength);
                    termBuffer.putLong(frameOffset + RESERVED_VALUE_OFFSET, reservedValue, LITTLE_ENDIAN);
                }

                frameLengthOrdered(termBuffer, frameOffset, frameLength);

                flags = 0;
                frameOffset += alignedLength;
                remaining -= bytesToWrite;
            }
            while (remaining > 0);
        }

        return (int)resultingOffset;
    }

    /**
     * Append a fragmented message to the the term buffer.
     * The message will be split up into fragments of MTU length minus header.
     *
     * @param header                for writing the default header.
     * @param vectors               to the buffers.
     * @param length                of the message as a sum of the vectors.
     * @param maxPayloadLength      that the message will be fragmented into.
     * @param reservedValueSupplier {@link ReservedValueSupplier} for the frame.
     * @param activeTermId          used for flow control.
     * @return the resulting offset of the term after the append on success otherwise {@link #FAILED}.
     */
    public int appendFragmentedMessage(
        final HeaderWriter header,
        final DirectBufferVector[] vectors,
        final int length,
        final int maxPayloadLength,
        final ReservedValueSupplier reservedValueSupplier,
        final int activeTermId)
    {
        final int numMaxPayloads = length / maxPayloadLength;
        final int remainingPayload = length % maxPayloadLength;
        final int lastFrameLength = remainingPayload > 0 ? align(remainingPayload + HEADER_LENGTH, FRAME_ALIGNMENT) : 0;
        final int requiredLength = (numMaxPayloads * (maxPayloadLength + HEADER_LENGTH)) + lastFrameLength;
        final UnsafeBuffer termBuffer = this.termBuffer;
        final int termLength = termBuffer.capacity();

        final long rawTail = getAndAddRawTail(requiredLength);
        final int termId = termId(rawTail);
        final long termOffset = rawTail & 0xFFFF_FFFFL;

        checkTerm(activeTermId, termId);

        long resultingOffset = termOffset + requiredLength;
        if (resultingOffset > termLength)
        {
            resultingOffset = handleEndOfLogCondition(termBuffer, termOffset, header, termLength, termId);
        }
        else
        {
            int frameOffset = (int)termOffset;
            byte flags = BEGIN_FRAG_FLAG;
            int remaining = length;
            int vectorIndex = 0;
            int vectorOffset = 0;

            do
            {
                final int bytesToWrite = Math.min(remaining, maxPayloadLength);
                final int frameLength = bytesToWrite + HEADER_LENGTH;
                final int alignedLength = align(frameLength, FRAME_ALIGNMENT);

                header.write(termBuffer, frameOffset, frameLength, termId);

                int bytesWritten = 0;
                int payloadOffset = frameOffset + HEADER_LENGTH;
                do
                {
                    final DirectBufferVector vector = vectors[vectorIndex];
                    final int vectorRemaining = vector.length - vectorOffset;
                    final int numBytes = Math.min(bytesToWrite - bytesWritten, vectorRemaining);

                    termBuffer.putBytes(payloadOffset, vector.buffer, vector.offset + vectorOffset, numBytes);

                    bytesWritten += numBytes;
                    payloadOffset += numBytes;
                    vectorOffset += numBytes;

                    if (vectorRemaining <= numBytes)
                    {
                        vectorIndex++;
                        vectorOffset = 0;
                    }
                }
                while (bytesWritten < bytesToWrite);

                if (remaining <= maxPayloadLength)
                {
                    flags |= END_FRAG_FLAG;
                }

                frameFlags(termBuffer, frameOffset, flags);

                if (null != reservedValueSupplier)
                {
                    final long reservedValue = reservedValueSupplier.get(termBuffer, frameOffset, frameLength);
                    termBuffer.putLong(frameOffset + RESERVED_VALUE_OFFSET, reservedValue, LITTLE_ENDIAN);
                }

                frameLengthOrdered(termBuffer, frameOffset, frameLength);

                flags = 0;
                frameOffset += alignedLength;
                remaining -= bytesToWrite;
            }
            while (remaining > 0);
        }

        return (int)resultingOffset;
    }

    private static void checkTerm(final int expectedTermId, final int termId)
    {
        if (termId != expectedTermId)
        {
            throw new AeronException(
                "action possibly delayed: expectedTermId=" + expectedTermId + " termId=" + termId);
        }
    }

    private static int handleEndOfLogCondition(
        final UnsafeBuffer termBuffer,
        final long termOffset,
        final HeaderWriter header,
        final int termLength,
        final int termId)
    {
        if (termOffset < termLength)
        {
            final int offset = (int)termOffset;
            final int paddingLength = termLength - offset;
            header.write(termBuffer, offset, paddingLength, termId);
            frameType(termBuffer, offset, PADDING_FRAME_TYPE);
            frameLengthOrdered(termBuffer, offset, paddingLength);
        }

        return FAILED;
    }

    private long getAndAddRawTail(final int alignedLength)
    {
        return UnsafeAccess.UNSAFE.getAndAddLong(null, tailAddressOffset, alignedLength);
    }
}


/*
 * Copyright 2014-2019 Real Logic Ltd.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package io.aeron.archive.client;

import io.aeron.ChannelUriStringBuilder;
import io.aeron.Publication;
import io.aeron.Subscription;
import io.aeron.archive.codecs.*;
import org.agrona.ExpandableArrayBuffer;
import org.agrona.concurrent.*;

import static io.aeron.archive.client.AeronArchive.Configuration.MESSAGE_TIMEOUT_DEFAULT_NS;

/**
 * Proxy class for encapsulating encoding and sending of control protocol messages to an archive.
 */
public class ArchiveProxy
{
    /**
     * Default number of retry attempts to be made at offering requests.
     */
    public static final int DEFAULT_RETRY_ATTEMPTS = 3;

    private final long connectTimeoutNs;
    private final int retryAttempts;
    private final IdleStrategy retryIdleStrategy;
    private final NanoClock nanoClock;

    private final ExpandableArrayBuffer buffer = new ExpandableArrayBuffer(256);
    private final Publication publication;
    private final MessageHeaderEncoder messageHeaderEncoder = new MessageHeaderEncoder();
    private final ConnectRequestEncoder connectRequestEncoder = new ConnectRequestEncoder();
    private final CloseSessionRequestEncoder closeSessionRequestEncoder = new CloseSessionRequestEncoder();
    private final StartRecordingRequestEncoder startRecordingRequestEncoder = new StartRecordingRequestEncoder();
    private final ReplayRequestEncoder replayRequestEncoder = new ReplayRequestEncoder();
    private final StopReplayRequestEncoder stopReplayRequestEncoder = new StopReplayRequestEncoder();
    private final StopRecordingRequestEncoder stopRecordingRequestEncoder = new StopRecordingRequestEncoder();
    private final StopRecordingSubscriptionRequestEncoder stopRecordingSubscriptionRequestEncoder =
        new StopRecordingSubscriptionRequestEncoder();
    private final ListRecordingsRequestEncoder listRecordingsRequestEncoder = new ListRecordingsRequestEncoder();
    private final ListRecordingsForUriRequestEncoder listRecordingsForUriRequestEncoder =
        new ListRecordingsForUriRequestEncoder();
    private final ListRecordingRequestEncoder listRecordingRequestEncoder = new ListRecordingRequestEncoder();
    private final ExtendRecordingRequestEncoder extendRecordingRequestEncoder = new ExtendRecordingRequestEncoder();
    private final RecordingPositionRequestEncoder recordingPositionRequestEncoder =
        new RecordingPositionRequestEncoder();
    private final TruncateRecordingRequestEncoder truncateRecordingRequestEncoder =
        new TruncateRecordingRequestEncoder();
    private final StopPositionRequestEncoder stopPositionRequestEncoder = new StopPositionRequestEncoder();
    private final FindLastMatchingRecordingRequestEncoder findLastMatchingRecordingRequestEncoder =
        new FindLastMatchingRecordingRequestEncoder();
    private final ListRecordingSubscriptionsRequestEncoder listRecordingSubscriptionsRequestEncoder =
        new ListRecordingSubscriptionsRequestEncoder();

    /**
     * Create a proxy with a {@link Publication} for sending control message requests.
     * <p>
     * This provides a default {@link IdleStrategy} of a {@link YieldingIdleStrategy} when offers are back pressured
     * with a defaults of {@link AeronArchive.Configuration#MESSAGE_TIMEOUT_DEFAULT_NS} and
     * {@link #DEFAULT_RETRY_ATTEMPTS}.
     *
     * @param publication publication for sending control messages to an archive.
     */
    public ArchiveProxy(final Publication publication)
    {
        this(
            publication,
            new YieldingIdleStrategy(),
            new SystemNanoClock(),
            MESSAGE_TIMEOUT_DEFAULT_NS,
            DEFAULT_RETRY_ATTEMPTS);
    }

    /**
     * Create a proxy with a {@link Publication} for sending control message requests.
     *
     * @param publication       publication for sending control messages to an archive.
     * @param retryIdleStrategy for what should happen between retry attempts at offering messages.
     * @param nanoClock         to be used for calculating checking deadlines.
     * @param connectTimeoutNs  for for connection requests.
     * @param retryAttempts     for offering control messages before giving up.
     */
    public ArchiveProxy(
        final Publication publication,
        final IdleStrategy retryIdleStrategy,
        final NanoClock nanoClock,
        final long connectTimeoutNs,
        final int retryAttempts)
    {
        this.publication = publication;
        this.retryIdleStrategy = retryIdleStrategy;
        this.nanoClock = nanoClock;
        this.connectTimeoutNs = connectTimeoutNs;
        this.retryAttempts = retryAttempts;
    }

    /**
     * Get the {@link Publication} used for sending control messages.
     *
     * @return the {@link Publication} used for sending control messages.
     */
    public Publication publication()
    {
        return publication;
    }

    /**
     * Connect to an archive on its control interface providing the response stream details.
     *
     * @param responseChannel  for the control message responses.
     * @param responseStreamId for the control message responses.
     * @param correlationId    for this request.
     * @return true if successfully offered otherwise false.
     */
    public boolean connect(final String responseChannel, final int responseStreamId, final long correlationId)
    {
        connectRequestEncoder
            .wrapAndApplyHeader(buffer, 0, messageHeaderEncoder)
            .correlationId(correlationId)
            .responseStreamId(responseStreamId)
            .version(AeronArchive.Configuration.SEMANTIC_VERSION)
            .responseChannel(responseChannel);

        return offerWithTimeout(connectRequestEncoder.encodedLength(), null);
    }

    /**
     * Try Connect to an archive on its control interface providing the response stream details. Only one attempt will
     * be made to offer the request.
     *
     * @param responseChannel  for the control message responses.
     * @param responseStreamId for the control message responses.
     * @param correlationId    for this request.
     * @return true if successfully offered otherwise false.
     */
    public boolean tryConnect(final String responseChannel, final int responseStreamId, final long correlationId)
    {
        connectRequestEncoder
            .wrapAndApplyHeader(buffer, 0, messageHeaderEncoder)
            .correlationId(correlationId)
            .responseStreamId(responseStreamId)
            .version(AeronArchive.Configuration.SEMANTIC_VERSION)
            .responseChannel(responseChannel);

        final int length = MessageHeaderEncoder.ENCODED_LENGTH + connectRequestEncoder.encodedLength();

        return publication.offer(buffer, 0, length) > 0;
    }

    /**
     * Connect to an archive on its control interface providing the response stream details.
     *
     * @param responseChannel    for the control message responses.
     * @param responseStreamId   for the control message responses.
     * @param correlationId      for this request.
     * @param aeronClientInvoker for aeron client conductor thread.
     * @return true if successfully offered otherwise false.
     */
    public boolean connect(
        final String responseChannel,
        final int responseStreamId,
        final long correlationId,
        final AgentInvoker aeronClientInvoker)
    {
        connectRequestEncoder
            .wrapAndApplyHeader(buffer, 0, messageHeaderEncoder)
            .correlationId(correlationId)
            .responseStreamId(responseStreamId)
            .version(AeronArchive.Configuration.SEMANTIC_VERSION)
            .responseChannel(responseChannel);

        return offerWithTimeout(connectRequestEncoder.encodedLength(), aeronClientInvoker);
    }

    /**
     * Close this control session with the archive.
     *
     * @param controlSessionId with the archive.
     * @return true if successfully offered otherwise false.
     */
    public boolean closeSession(final long controlSessionId)
    {
        closeSessionRequestEncoder
            .wrapAndApplyHeader(buffer, 0, messageHeaderEncoder)
            .controlSessionId(controlSessionId);

        return offer(closeSessionRequestEncoder.encodedLength());
    }

    /**
     * Start recording streams for a given channel and stream id pairing.
     *
     * @param channel          to be recorded.
     * @param streamId         to be recorded.
     * @param sourceLocation   of the publication to be recorded.
     * @param correlationId    for this request.
     * @param controlSessionId for this request.
     * @return true if successfully offered otherwise false.
     */
    public boolean startRecording(
        final String channel,
        final int streamId,
        final SourceLocation sourceLocation,
        final long correlationId,
        final long controlSessionId)
    {
        startRecordingRequestEncoder
            .wrapAndApplyHeader(buffer, 0, messageHeaderEncoder)
            .controlSessionId(controlSessionId)
            .correlationId(correlationId)
            .streamId(streamId)
            .sourceLocation(sourceLocation)
            .channel(channel);

        return offer(startRecordingRequestEncoder.encodedLength());
    }

    /**
     * Stop an active recording.
     *
     * @param channel          to be stopped.
     * @param streamId         to be stopped.
     * @param correlationId    for this request.
     * @param controlSessionId for this request.
     * @return true if successfully offered otherwise false.
     */
    public boolean stopRecording(
        final String channel,
        final int streamId,
        final long correlationId,
        final long controlSessionId)
    {
        stopRecordingRequestEncoder
            .wrapAndApplyHeader(buffer, 0, messageHeaderEncoder)
            .controlSessionId(controlSessionId)
            .correlationId(correlationId)
            .streamId(streamId)
            .channel(channel);

        return offer(stopRecordingRequestEncoder.encodedLength());
    }

    /**
     * Stop an active recording by the {@link Subscription#registrationId()} it was registered with.
     *
     * @param subscriptionId   that identifies the subscription in the archive doing the recording.
     * @param correlationId    for this request.
     * @param controlSessionId for this request.
     * @return true if successfully offered otherwise false.
     */
    public boolean stopRecording(
        final long subscriptionId,
        final long correlationId,
        final long controlSessionId)
    {
        stopRecordingSubscriptionRequestEncoder
            .wrapAndApplyHeader(buffer, 0, messageHeaderEncoder)
            .controlSessionId(controlSessionId)
            .correlationId(correlationId)
            .subscriptionId(subscriptionId);

        return offer(stopRecordingSubscriptionRequestEncoder.encodedLength());
    }

    /**
     * Replay a recording from a given position.
     *
     * @param recordingId      to be replayed.
     * @param position         from which the replay should be started.
     * @param length           of the stream to be replayed. Use {@link Long#MAX_VALUE} to follow a live stream.
     * @param replayChannel    to which the replay should be sent.
     * @param replayStreamId   to which the replay should be sent.
     * @param correlationId    for this request.
     * @param controlSessionId for this request.
     * @return true if successfully offered otherwise false.
     */
    public boolean replay(
        final long recordingId,
        final long position,
        final long length,
        final String replayChannel,
        final int replayStreamId,
        final long correlationId,
        final long controlSessionId)
    {
        replayRequestEncoder
            .wrapAndApplyHeader(buffer, 0, messageHeaderEncoder)
            .controlSessionId(controlSessionId)
            .correlationId(correlationId)
            .recordingId(recordingId)
            .position(position)
            .length(length)
            .replayStreamId(replayStreamId)
            .replayChannel(replayChannel);

        return offer(replayRequestEncoder.encodedLength());
    }

    /**
     * Stop an existing replay session.
     *
     * @param replaySessionId  that should be stopped.
     * @param correlationId    for this request.
     * @param controlSessionId for this request.
     * @return true if successfully offered otherwise false.
     */
    public boolean stopReplay(final long replaySessionId, final long correlationId, final long controlSessionId)
    {
        stopReplayRequestEncoder
            .wrapAndApplyHeader(buffer, 0, messageHeaderEncoder)
            .controlSessionId(controlSessionId)
            .correlationId(correlationId)
            .replaySessionId(replaySessionId);

        return offer(replayRequestEncoder.encodedLength());
    }

    /**
     * List a range of recording descriptors.
     *
     * @param fromRecordingId  at which to begin listing.
     * @param recordCount      for the number of descriptors to be listed.
     * @param correlationId    for this request.
     * @param controlSessionId for this request.
     * @return true if successfully offered otherwise false.
     */
    public boolean listRecordings(
        final long fromRecordingId, final int recordCount, final long correlationId, final long controlSessionId)
    {
        listRecordingsRequestEncoder
            .wrapAndApplyHeader(buffer, 0, messageHeaderEncoder)
            .controlSessionId(controlSessionId)
            .correlationId(correlationId)
            .fromRecordingId(fromRecordingId)
            .recordCount(recordCount);

        return offer(listRecordingsRequestEncoder.encodedLength());
    }

    /**
     * List a range of recording descriptors which match a channel URI fragment and stream id.
     *
     * @param fromRecordingId  at which to begin listing.
     * @param recordCount      for the number of descriptors to be listed.
     * @param channelFragment  to match recordings on from the original channel URI in the archive descriptor.
     * @param streamId         to match recordings on.
     * @param correlationId    for this request.
     * @param controlSessionId for this request.
     * @return true if successfully offered otherwise false.
     */
    public boolean listRecordingsForUri(
        final long fromRecordingId,
        final int recordCount,
        final String channelFragment,
        final int streamId,
        final long correlationId,
        final long controlSessionId)
    {
        listRecordingsForUriRequestEncoder
            .wrapAndApplyHeader(buffer, 0, messageHeaderEncoder)
            .controlSessionId(controlSessionId)
            .correlationId(correlationId)
            .fromRecordingId(fromRecordingId)
            .recordCount(recordCount)
            .streamId(streamId)
            .channel(channelFragment);

        return offer(listRecordingsForUriRequestEncoder.encodedLength());
    }

    /**
     * List a recording descriptor for a given recording id.
     *
     * @param recordingId      at which to begin listing.
     * @param correlationId    for this request.
     * @param controlSessionId for this request.
     * @return true if successfully offered otherwise false.
     */
    public boolean listRecording(final long recordingId, final long correlationId, final long controlSessionId)
    {
        listRecordingRequestEncoder
            .wrapAndApplyHeader(buffer, 0, messageHeaderEncoder)
            .controlSessionId(controlSessionId)
            .correlationId(correlationId)
            .recordingId(recordingId);

        return offer(listRecordingRequestEncoder.encodedLength());
    }

    /**
     * Extend an existing, non-active, recorded stream for a the same channel and stream id.
     *
     * The channel must be configured for the initial position from which it will be extended. This can be done
     * with {@link ChannelUriStringBuilder#initialPosition(long, int, int)}. The details required to initialise can
     * be found by calling {@link #listRecording(long, long, long)}.
     *
     * @param channel          to be recorded.
     * @param streamId         to be recorded.
     * @param sourceLocation   of the publication to be recorded.
     * @param recordingId      to be extended.
     * @param correlationId    for this request.
     * @param controlSessionId for this request.
     * @return true if successfully offered otherwise false.
     */
    public boolean extendRecording(
        final String channel,
        final int streamId,
        final SourceLocation sourceLocation,
        final long recordingId,
        final long correlationId,
        final long controlSessionId)
    {
        extendRecordingRequestEncoder
            .wrapAndApplyHeader(buffer, 0, messageHeaderEncoder)
            .controlSessionId(controlSessionId)
            .correlationId(correlationId)
            .recordingId(recordingId)
            .streamId(streamId)
            .sourceLocation(sourceLocation)
            .channel(channel);

        return offer(extendRecordingRequestEncoder.encodedLength());
    }

    /**
     * Get the recorded position of an active recording.
     *
     * @param recordingId      of the active recording that the position is being requested for.
     * @param correlationId    for this request.
     * @param controlSessionId for this request.
     * @return true if successfully offered otherwise false.
     */
    public boolean getRecordingPosition(final long recordingId, final long correlationId, final long controlSessionId)
    {
        recordingPositionRequestEncoder
            .wrapAndApplyHeader(buffer, 0, messageHeaderEncoder)
            .controlSessionId(controlSessionId)
            .correlationId(correlationId)
            .recordingId(recordingId);

        return offer(recordingPositionRequestEncoder.encodedLength());
    }

    /**
     * Truncate a stopped recording to a given position that is less than the stopped position. The provided position
     * must be on a fragment boundary. Truncating a recording to the start position effectively deletes the recording.
     *
     * @param recordingId      of the stopped recording to be truncated.
     * @param position         to which the recording will be truncated.
     * @param correlationId    for this request.
     * @param controlSessionId for this request.
     * @return true if successfully offered otherwise false.
     */
    public boolean truncateRecording(
        final long recordingId, final long position, final long correlationId, final long controlSessionId)
    {
        truncateRecordingRequestEncoder
            .wrapAndApplyHeader(buffer, 0, messageHeaderEncoder)
            .controlSessionId(controlSessionId)
            .correlationId(correlationId)
            .recordingId(recordingId)
            .position(position);

        return offer(truncateRecordingRequestEncoder.encodedLength());
    }

    /**
     * Get the stop position of a recording.
     *
     * @param recordingId      of the recording that the stop position is being requested for.
     * @param correlationId    for this request.
     * @param controlSessionId for this request.
     * @return true if successfully offered otherwise false.
     */
    public boolean getStopPosition(final long recordingId, final long correlationId, final long controlSessionId)
    {
        stopPositionRequestEncoder
            .wrapAndApplyHeader(buffer, 0, messageHeaderEncoder)
            .controlSessionId(controlSessionId)
            .correlationId(correlationId)
            .recordingId(recordingId);

        return offer(stopPositionRequestEncoder.encodedLength());
    }

    /**
     * Find the last recording that matches the given criteria.
     *
     * @param minRecordingId   to search back to.
     * @param channelFragment  for a contains match on the original channel stored with the archive descriptor.
     * @param streamId         of the recording to match.
     * @param sessionId        of the recording to match.
     * @param correlationId    for this request.
     * @param controlSessionId for this request.
     * @return true if successfully offered otherwise false.
     */
    public boolean findLastMatchingRecording(
        final long minRecordingId,
        final String channelFragment,
        final int streamId,
        final int sessionId,
        final long correlationId,
        final long controlSessionId)
    {
        findLastMatchingRecordingRequestEncoder
            .wrapAndApplyHeader(buffer, 0, messageHeaderEncoder)
            .controlSessionId(controlSessionId)
            .correlationId(correlationId)
            .minRecordingId(minRecordingId)
            .sessionId(sessionId)
            .streamId(streamId)
            .channel(channelFragment);

        return offer(findLastMatchingRecordingRequestEncoder.encodedLength());
    }

    /**
     * List registered subscriptions in the archive which have been used to record streams.
     *
     * @param pseudoIndex       in the list of active recording subscriptions.
     * @param subscriptionCount for the number of descriptors to be listed.
     * @param channelFragment   for a contains match on the stripped channel used with the registered subscription.
     * @param streamId          for the subscription.
     * @param applyStreamId     when matching.
     * @param correlationId     for this request.
     * @param controlSessionId  for this request.
     * @return true if successfully offered otherwise false.
     */
    public boolean listRecordingSubscriptions(
        final int pseudoIndex,
        final int subscriptionCount,
        final String channelFragment,
        final int streamId,
        final boolean applyStreamId,
        final long correlationId,
        final long controlSessionId)
    {
        listRecordingSubscriptionsRequestEncoder
            .wrapAndApplyHeader(buffer, 0, messageHeaderEncoder)
            .controlSessionId(controlSessionId)
            .correlationId(correlationId)
            .pseudoIndex(pseudoIndex)
            .subscriptionCount(subscriptionCount)
            .applyStreamId(applyStreamId ? BooleanType.TRUE : BooleanType.FALSE)
            .streamId(streamId)
            .channel(channelFragment);

        return offer(listRecordingSubscriptionsRequestEncoder.encodedLength());
    }

    private boolean offer(final int length)
    {
        retryIdleStrategy.reset();

        int attempts = retryAttempts;
        while (true)
        {
            final long result = publication.offer(buffer, 0, MessageHeaderEncoder.ENCODED_LENGTH + length);
            if (result > 0)
            {
                return true;
            }

            if (result == Publication.CLOSED)
            {
                throw new ArchiveException("connection to the archive has been closed");
            }

            if (result == Publication.NOT_CONNECTED)
            {
                throw new ArchiveException("connection to the archive is no longer available");
            }

            if (result == Publication.MAX_POSITION_EXCEEDED)
            {
                throw new ArchiveException("offer failed due to max position being reached");
            }

            if (--attempts <= 0)
            {
                return false;
            }

            retryIdleStrategy.idle();
        }
    }

    private boolean offerWithTimeout(final int length, final AgentInvoker aeronClientInvoker)
    {
        retryIdleStrategy.reset();

        final long deadlineNs = nanoClock.nanoTime() + connectTimeoutNs;
        while (true)
        {
            final long result = publication.offer(buffer, 0, MessageHeaderEncoder.ENCODED_LENGTH + length);
            if (result > 0)
            {
                return true;
            }

            if (result == Publication.CLOSED)
            {
                throw new ArchiveException("connection to the archive has been closed");
            }

            if (result == Publication.MAX_POSITION_EXCEEDED)
            {
                throw new ArchiveException("offer failed due to max position being reached");
            }

            if (deadlineNs - nanoClock.nanoTime() < 0)
            {
                return false;
            }

            if (null != aeronClientInvoker)
            {
                aeronClientInvoker.invoke();
            }

            retryIdleStrategy.idle();
        }
    }
}