/*
 * This file and its contents are licensed under the Timescale License.
 * Please see the included NOTICE for copyright information and
 * LICENSE-TIMESCALE for a copy of the license.
 */
#include <postgres.h>

#include <access/htup.h>
#include <access/htup_details.h>
#include <access/xact.h>
#include <extension.h>
#include <fmgr.h>
#include <funcapi.h>
#include <hypertable_cache.h>
#include <miscadmin.h>
#include <nodes/makefuncs.h>
#include <nodes/memnodes.h>
#include <parser/parse_func.h>
#include <scan_iterator.h>
#include <scanner.h>
#include <storage/lockdefs.h>
#include <time_bucket.h>
#include <time_utils.h>
#include <utils.h>
#include <utils/builtins.h>
#include <utils/elog.h>
#include <utils/lsyscache.h>
#include <utils/memutils.h>
#include <utils/palloc.h>
#include <utils/snapmgr.h>
#include <utils/tuplestore.h>

#include "cache.h"
#include "continuous_aggs/invalidation_threshold.h"
#include "continuous_aggs/materialize.h"
#include "guc.h"
#include "invalidation.h"
#include "refresh.h"
#include "ts_catalog/catalog.h"
#include "ts_catalog/continuous_agg.h"
#include "ts_catalog/continuous_aggs_watermark.h"

/*
 * Invalidation processing for continuous aggregates.
 *
 * Invalidations track the regions/intervals [start, end] of a continuous
 * aggregate that are out-of-date relative to the source hypertable on top of
 * which the aggregate is defined. When a continuous aggregate is out-of-date
 * across one or more regions, it can be refreshed with a window covering
 * those regions in order to bring it up-to-date with the source data again.
 *
 * Invalidations are generated by mutations on the source hypertable (INSERT,
 * DELETE, UPDATE, TRUNCATE, drop_chunks, etc.) and are initially written to a
 * hypertable invalidation log [hypertable_id, start, end].
 *
 * When a continuous aggregate is refreshed, invalidations are moved from the
 * hypertable invalidation log to a continuous aggregate invalidation log,
 * where each original entry creates one new entry per continuous aggregate
 * [cagg_id, start, end]. Thus, if one continuous aggregate is refreshed but
 * not others, then only the invalidations for the refreshed aggregate are
 * processed.
 *
 * Simplified, invalidations move through the following stages:
 *
 * insert_trigger => hypertable_inval_log => cagg_inval_log => refreshing
 *
 * Thus, invalidations are generated by mutations and are processed and used
 * as input for refreshing the a continuous aggregate.
 *
 * Invalidations can overlap or be duplicates. Therefore, invalidations are
 * merged during processing to reduce the number of entries in the logs. This
 * typically happens during a refresh of a continuous aggregate, which also
 * cuts invalidations along the refresh window. The cutting will leave some
 * parts of entries in the invalidation log while the entries that fall
 * within the refresh window are stored in an invalidation store and used for
 * refreshing:
 *
 *       |-------------|    refresh window
 *
 *   |-----|  |---| |----|  invalidations
 *
 *             =>
 *
 *   |---|             |--| invalidations that remain in the log
 *
 *       |-|  |---| |--|    invalidations that are used for refreshing
 *
 */

/*
 * Processing state used while processing the materialization invalidation log
 * and refreshing the continuous aggregate.
 */
typedef struct ContinuousAggInvalidationState
{
	const ContinuousAgg *cagg;
	MemoryContext per_tuple_mctx;
	Relation cagg_log_rel;
	Snapshot snapshot;
} ContinuousAggInvalidationState;

/*
 * Processing state used while moving invalidations from hypertable
 * invalidation log to materialization invalidation log.
 */
typedef struct HypertableInvalidationState
{
	int32 hypertable_id;
	Oid dimtype; /* Type of the underlying hypertable's bucketed attribute */
	const ContinuousAggInfo *all_caggs;
	MemoryContext per_tuple_mctx;
	Relation cagg_log_rel;
	Snapshot snapshot;
} HypertableInvalidationState;

typedef enum ContinuousAggTableType
{
	HYPER_INVALIDATION_LOG,
	CAGG_INVALIDATION_LOG,
} ContinuousAggTableType;

static Relation open_cagg_table(ContinuousAggTableType type, LOCKMODE lockmode);
static void hypertable_invalidation_scan_init(ScanIterator *iterator, int32 hyper_id,
											  LOCKMODE lockmode);
static bool insert_invalidation_entry(Relation cagg_log_rel, const Invalidation *invalidation);
static bool write_invalidation_entry(Relation cagg_log_rel, const Invalidation *invalidation,
									 ItemPointer update_tid);
static void invalidation_entry_set(Invalidation *inner_range, int32 hyper_id,
								   int64 lowest_modified_value, int64 greatest_modified_value);
static void invalidation_entry_reset(Invalidation *entry);
static void
invalidation_entry_set_from_hyper_invalidation(Invalidation *entry, const TupleInfo *ti,
											   int32 hyper_id, Oid dimtype,
											   const ContinuousAggBucketFunction *bucket_function);
static void
invalidation_entry_set_from_cagg_invalidation(Invalidation *entry, const TupleInfo *ti, Oid dimtype,
											  const ContinuousAggBucketFunction *bucket_function);
static bool invalidations_can_be_merged(const Invalidation *a, const Invalidation *b);
static bool invalidation_entry_try_merge(Invalidation *entry, const Invalidation *newentry);
static void insert_new_cagg_invalidation(const HypertableInvalidationState *state,
										 const Invalidation *entry, int32 cagg_hyper_id);
static void move_invalidations_from_hyper_to_cagg_log(const HypertableInvalidationState *state);
static void cagg_invalidations_scan_by_hypertable_init(ScanIterator *iterator, int32 cagg_hyper_id,
													   LOCKMODE lockmode, int64 window_end);
static ScanFilterResult cagg_invalidation_window_end_filter(const TupleInfo *ti, void *data);
static Invalidation cut_cagg_invalidation(const ContinuousAggInvalidationState *state,
										  const InternalTimeRange *refresh_window,
										  const Invalidation *entry);
static Invalidation cut_cagg_invalidation_and_compute_inner_range(
	const ContinuousAggInvalidationState *state, const InternalTimeRange *refresh_window,
	const Invalidation *mergedentry, const Invalidation *current_inner_range);
static void process_cagg_invalidations_for_refresh(const ContinuousAggInvalidationState *state,
												   const InternalTimeRange *refresh_window);
static void cagg_invalidation_state_init(ContinuousAggInvalidationState *state,
										 const ContinuousAgg *cagg);
static void cagg_invalidation_state_cleanup(const ContinuousAggInvalidationState *state);

static Relation
open_cagg_table(ContinuousAggTableType type, LOCKMODE lockmode)
{
	static const CatalogTable logmappings[] = {
		[HYPER_INVALIDATION_LOG] = CONTINUOUS_AGGS_HYPERTABLE_INVALIDATION_LOG,
		[CAGG_INVALIDATION_LOG] = CONTINUOUS_AGGS_MATERIALIZATION_INVALIDATION_LOG,
	};
	Catalog *catalog = ts_catalog_get();
	Oid relid = catalog_get_table_id(catalog, logmappings[type]);

	return table_open(relid, lockmode);
}

static void
hypertable_invalidation_scan_init(ScanIterator *iterator, int32 hyper_id, LOCKMODE lockmode)
{
	*iterator = ts_scan_iterator_create(CONTINUOUS_AGGS_HYPERTABLE_INVALIDATION_LOG,
										lockmode,
										CurrentMemoryContext);
	iterator->ctx.index = catalog_get_index(ts_catalog_get(),
											CONTINUOUS_AGGS_HYPERTABLE_INVALIDATION_LOG,
											CONTINUOUS_AGGS_HYPERTABLE_INVALIDATION_LOG_IDX);
	ts_scan_iterator_scan_key_init(
		iterator,
		Anum_continuous_aggs_hypertable_invalidation_log_idx_hypertable_id,
		BTEqualStrategyNumber,
		F_INT4EQ,
		Int32GetDatum(hyper_id));
}

HeapTuple
create_invalidation_tup(const TupleDesc tupdesc, int32 cagg_hyper_id, int64 start, int64 end)
{
	Datum values[Natts_continuous_aggs_materialization_invalidation_log] = { 0 };
	bool isnull[Natts_continuous_aggs_materialization_invalidation_log] = { false };

	values[AttrNumberGetAttrOffset(
		Anum_continuous_aggs_materialization_invalidation_log_materialization_id)] =
		Int32GetDatum(cagg_hyper_id);
	values[AttrNumberGetAttrOffset(
		Anum_continuous_aggs_materialization_invalidation_log_lowest_modified_value)] =
		Int64GetDatum(start);
	values[AttrNumberGetAttrOffset(
		Anum_continuous_aggs_materialization_invalidation_log_greatest_modified_value)] =
		Int64GetDatum(end);

	return heap_form_tuple(tupdesc, values, isnull);
}

/*
 * Add an entry to the continuous aggregate invalidation log.
 */
void
invalidation_cagg_log_add_entry(int32 cagg_hyper_id, int64 start, int64 end)
{
	Relation rel = open_cagg_table(CAGG_INVALIDATION_LOG, RowExclusiveLock);
	CatalogSecurityContext sec_ctx;
	HeapTuple tuple;

	Assert(start <= end);
	tuple = create_invalidation_tup(RelationGetDescr(rel), cagg_hyper_id, start, end);
	ts_catalog_database_info_become_owner(ts_catalog_database_info_get(), &sec_ctx);
	ts_catalog_insert_only(rel, tuple);
	ts_catalog_restore_user(&sec_ctx);
	heap_freetuple(tuple);
	table_close(rel, NoLock);
}

void
invalidation_hyper_log_add_entry(int32 hyper_id, int64 start, int64 end)
{
	Relation rel = open_cagg_table(HYPER_INVALIDATION_LOG, RowExclusiveLock);
	CatalogSecurityContext sec_ctx;
	Datum values[Natts_continuous_aggs_hypertable_invalidation_log];
	bool nulls[Natts_continuous_aggs_hypertable_invalidation_log] = { false };

	Assert(start <= end);
	values[AttrNumberGetAttrOffset(
		Anum_continuous_aggs_hypertable_invalidation_log_hypertable_id)] = Int32GetDatum(hyper_id);
	values[AttrNumberGetAttrOffset(
		Anum_continuous_aggs_hypertable_invalidation_log_lowest_modified_value)] =
		Int64GetDatum(start);
	values[AttrNumberGetAttrOffset(
		Anum_continuous_aggs_hypertable_invalidation_log_greatest_modified_value)] =
		Int64GetDatum(end);

	ts_catalog_database_info_become_owner(ts_catalog_database_info_get(), &sec_ctx);
	ts_catalog_insert_values(rel, RelationGetDescr(rel), values, nulls);
	ts_catalog_restore_user(&sec_ctx);
	table_close(rel, NoLock);
	elog(DEBUG1,
		 "hypertable log for hypertable %d added entry [" INT64_FORMAT ", " INT64_FORMAT "]",
		 hyper_id,
		 start,
		 end);
}

/*
 * Invalidate one or more continuous aggregates.
 *
 * Add an invalidation in the given range. The invalidation is added either to
 * the hypertable invalidation log or the continuous aggregate invalidation
 * log depending on the type of the given hypertable. If the hypertable is a
 * "raw" hypertable (i.e., one that has one or more continuous aggregates), the
 * entry is added to the hypertable invalidation log and will invalidate all
 * the associated continuous aggregates. If the hypertable is instead an
 * materialized hypertable, the entry is added to the cagg invalidation log
 * and only invalidates the continuous aggregate owning that materialized
 * hypertable.
 */
void
continuous_agg_invalidate_raw_ht(const Hypertable *raw_ht, int64 start, int64 end)
{
	Assert(raw_ht != NULL);

	invalidation_hyper_log_add_entry(raw_ht->fd.id, start, end);
}

void
continuous_agg_invalidate_mat_ht(const Hypertable *raw_ht, const Hypertable *mat_ht, int64 start,
								 int64 end)
{
	Assert((raw_ht != NULL) && (mat_ht != NULL));

	invalidation_cagg_log_add_entry(mat_ht->fd.id, start, end);
}

typedef enum InvalidationResult
{
	INVAL_NOMATCH,
	INVAL_FULLY_MATCH,
	INVAL_CUT,
} InvalidationResult;

static inline bool
IsValidInvalidation(const Invalidation *invalidation)
{
	Assert(invalidation->lowest_modified_value <= invalidation->greatest_modified_value);
	return invalidation->hyper_id != INVALID_HYPERTABLE_ID &&
		   invalidation->lowest_modified_value <= invalidation->greatest_modified_value;
}

static bool
write_invalidation_entry(Relation cagg_log_rel, const Invalidation *invalidation,
						 ItemPointer update_tid)
{
	CatalogSecurityContext sec_ctx;
	HeapTuple tup;

	if (!IsValidInvalidation(invalidation))
	{
		return false;
	}

	tup = create_invalidation_tup(RelationGetDescr(cagg_log_rel),
								  invalidation->hyper_id,
								  invalidation->lowest_modified_value,
								  invalidation->greatest_modified_value);
	ts_catalog_database_info_become_owner(ts_catalog_database_info_get(), &sec_ctx);
	if (ItemPointerIsValid(update_tid))
	{
		ts_catalog_update_tid_only(cagg_log_rel, update_tid, tup);
	}
	else
	{
		ts_catalog_insert_only(cagg_log_rel, tup);
	}
	ts_catalog_restore_user(&sec_ctx);
	heap_freetuple(tup);

	return true;
}

static bool
insert_invalidation_entry(Relation cagg_log_rel, const Invalidation *invalidation)
{
	return write_invalidation_entry(cagg_log_rel, invalidation, NULL);
}

static void
invalidation_entry_set(Invalidation *inner_range, int32 hyper_id, int64 lowest_modified_value,
					   int64 greatest_modified_value)
{
	MemSet(inner_range, 0, sizeof(*inner_range));
	inner_range->hyper_id = hyper_id;
	inner_range->lowest_modified_value = lowest_modified_value;
	inner_range->greatest_modified_value = greatest_modified_value;
}

/*
 * Try to cut an invalidation against the refresh window.
 *
 * If an invalidation entry overlaps with the refresh window, it needs
 * additional processing: it is either cut, deleted, or left unmodified.
 *
 * The part(s) of the invalidation that are outside the refresh window after
 * the cut will remain in the log. The part of the invalidation that fits
 * within the window is returned as the "inner range".
 *
 * Note that the refresh window is exclusive in the end while invalidations
 * are inclusive.
 */
static InvalidationResult
cut_invalidation_along_refresh_window(const ContinuousAggInvalidationState *state,
									  const Invalidation *invalidation,
									  const InternalTimeRange *refresh_window,
									  Invalidation *inner_range)
{
	int32 cagg_hyper_id = state->cagg->data.mat_hypertable_id;
	InvalidationResult result = INVAL_NOMATCH;
	Invalidation lower_range = { 0 };
	Invalidation upper_range = { 0 };

	Assert(inner_range != NULL);

	/* Entry is completely enclosed by the refresh window */
	if (invalidation->lowest_modified_value >= refresh_window->start &&
		invalidation->greatest_modified_value < refresh_window->end)
	{
		/*
		 * Entry completely enclosed — left in the log for Txn 3 to process:
		 *
		 * [---------------)
		 *     [+++++]
		 */

		result = INVAL_FULLY_MATCH;
	}
	else
	{
		if (invalidation->lowest_modified_value < refresh_window->start &&
			invalidation->greatest_modified_value >= refresh_window->start)
		{
			/*
			 * Need to cut in right end:
			 *
			 *     [------)
			 * [++++++]
			 *
			 * [++]
			 */
			invalidation_entry_set(&lower_range,
								   cagg_hyper_id,
								   invalidation->lowest_modified_value,
								   refresh_window->start - 1);
			invalidation_entry_set(inner_range,
								   cagg_hyper_id,
								   refresh_window->start,
								   /* Refresh window not exclusive at end */
								   MIN(refresh_window->end - 1,
									   invalidation->greatest_modified_value));
			result = INVAL_CUT;
		}

		if (invalidation->lowest_modified_value < refresh_window->end &&
			invalidation->greatest_modified_value >= refresh_window->end)
		{
			/*
			 * If the invalidation is already cut on the left above, the inner_range is set and
			 * will be reset here. The assert prevents from losing information from the inner_range.
			 */
			Assert((result == INVAL_CUT &&
					inner_range->lowest_modified_value == refresh_window->start) ||
				   result == INVAL_NOMATCH);

			/*
			 * Need to cut in left end:
			 *
			 * [------)
			 *    [++++++++]
			 *
			 *        [++++]
			 */
			invalidation_entry_set(&upper_range,
								   cagg_hyper_id,
								   refresh_window->end,
								   invalidation->greatest_modified_value);
			invalidation_entry_set(inner_range,
								   cagg_hyper_id,
								   MAX(invalidation->lowest_modified_value, refresh_window->start),
								   /* Refresh window exclusive at end */
								   refresh_window->end - 1);
			result = INVAL_CUT;
		}
	}

	/* Write outside parts back to the cagg invalidation log */
	if (result == INVAL_CUT)
	{
		/*
		 * We reuse (update) the original tuple slot for the first outside piece.
		 * If no TID is available (entry came from hyper log), insert instead.
		 * Any second outside piece is always a fresh insert.
		 */
		ItemPointerData tid = invalidation->tid;

		if (IsValidInvalidation(&lower_range))
		{
			write_invalidation_entry(state->cagg_log_rel,
									 &lower_range,
									 ItemPointerIsValid(&tid) ? &tid : NULL);
			ItemPointerSetInvalid(&tid); /* TID consumed — upper must be a fresh insert */
		}

		if (IsValidInvalidation(&upper_range))
		{
			write_invalidation_entry(state->cagg_log_rel,
									 &upper_range,
									 ItemPointerIsValid(&tid) ? &tid : NULL);
		}
	}

	return result;
}

static void
invalidation_entry_reset(Invalidation *entry)
{
	MemSet(entry, 0, sizeof(Invalidation));
}

/*
 * Expand an invalidation to bucket boundaries.
 *
 * Since a refresh always materializes full buckets, we can safely expand an
 * invalidation to bucket boundaries and in the process merge a lot more
 * invalidations.
 */
void
invalidation_expand_to_bucket_boundaries(Invalidation *inv, Oid time_type_oid,
										 const ContinuousAggBucketFunction *bucket_function)
{
	const int64 time_dimension_min = ts_time_get_min(time_type_oid);
	const int64 time_dimension_max = ts_time_get_max(time_type_oid);
	int64 min_bucket_start;
	int64 max_bucket_end;

	if (bucket_function->bucket_fixed_interval == false)
	{
		ts_compute_circumscribed_bucketed_refresh_window_variable(&inv->lowest_modified_value,
																  &inv->greatest_modified_value,
																  bucket_function);
		/* ts_compute_circumscribed_bucketed_refresh_window_variable returns the start of the
		 * next bucket as the end (exclusive). Since invalidations are inclusive at both ends,
		 * subtract 1 to get the last value of the current bucket (inclusive).
		 * Don't adjust infinity values. */
		if (inv->greatest_modified_value != INVAL_POS_INFINITY &&
			inv->greatest_modified_value != INVAL_NEG_INFINITY)
		{
			inv->greatest_modified_value = int64_saturating_sub(inv->greatest_modified_value, 1);
		}
		return;
	}

	int64 bucket_width = ts_continuous_agg_fixed_bucket_width(bucket_function);
	Assert(bucket_width > 0);

	NullableDatum offset = INIT_NULL_DATUM;
	NullableDatum origin = INIT_NULL_DATUM;
	fill_bucket_offset_origin(bucket_function, time_type_oid, &offset, &origin);

	/* Compute the start of the "first" bucket for the type. The min value
	 * must be at the start of the "first" bucket or somewhere in the
	 * bucket. If the min value falls on the exact start of the bucket we are
	 * good. Otherwise, we need to move to the next full bucket. */
	min_bucket_start = ts_time_saturating_add(time_dimension_min, bucket_width - 1, time_type_oid);
	min_bucket_start = ts_time_bucket_by_type(bucket_width, min_bucket_start, time_type_oid);

	/* Compute the end of the "last" bucket for the time type. Remember that
	 * invalidations are inclusive, so the "greatest" value should be the last
	 * value of the last full bucket. Either the max value is already the last
	 * value of the last bucket, or we need to return the last value of the
	 * previous full bucket.  */
	max_bucket_end = ts_time_bucket_by_type(bucket_width, time_dimension_max, time_type_oid);

	/* Check if the max value was already the last value of the last bucket */
	if (ts_time_saturating_add(max_bucket_end, bucket_width - 1, time_type_oid) ==
		time_dimension_max)
	{
		max_bucket_end = time_dimension_max;
	}
	else
	{
		/* The last bucket was partial. To get the end of previous bucket, we
		 * need to move one step down from the partial last bucket. */
		max_bucket_end = ts_time_saturating_sub(max_bucket_end, 1, time_type_oid);
	}

	if (inv->lowest_modified_value < min_bucket_start)
	{
		/* Below the min bucket, so treat as invalid to -infinity. */
		inv->lowest_modified_value = INVAL_NEG_INFINITY;
	}
	else if (inv->lowest_modified_value > max_bucket_end)
	{
		/* Above the max bucket, so treat as invalid to +infinity. */
		inv->lowest_modified_value = INVAL_POS_INFINITY;
	}
	else
	{
		inv->lowest_modified_value = ts_time_bucket_by_type_extended(bucket_width,
																	 inv->lowest_modified_value,
																	 time_type_oid,
																	 offset,
																	 origin);
	}

	if (inv->greatest_modified_value < min_bucket_start)
	{
		/* Below the min bucket, so treat as invalid to -infinity. */
		inv->greatest_modified_value = INVAL_NEG_INFINITY;
	}
	else if (inv->greatest_modified_value > max_bucket_end)
	{
		/* Above the max bucket, so treat as invalid to +infinity. */
		inv->greatest_modified_value = INVAL_POS_INFINITY;
	}
	else
	{
		inv->greatest_modified_value = ts_time_bucket_by_type_extended(bucket_width,
																	   inv->greatest_modified_value,
																	   time_type_oid,
																	   offset,
																	   origin);
		inv->greatest_modified_value =
			ts_time_saturating_add(inv->greatest_modified_value, bucket_width - 1, time_type_oid);
	}
}

/*
 * Macro to set an Invalidation from a tuple. The tuple can either have the
 * format of the hypertable invalidation log or the continuous aggregate
 * invalidation log (as determined by the type parameter).
 */
#define INVALIDATION_ENTRY_SET(entry, ti, hypertable_id, type)                                     \
	do                                                                                             \
	{                                                                                              \
		bool should_free;                                                                          \
		HeapTuple tuple = ts_scanner_fetch_heap_tuple(ti, false, &should_free);                    \
		type form;                                                                                 \
		form = (type) GETSTRUCT(tuple);                                                            \
		(entry)->hyper_id = form->hypertable_id;                                                   \
		(entry)->lowest_modified_value = form->lowest_modified_value;                              \
		(entry)->greatest_modified_value = form->greatest_modified_value;                          \
		(entry)->is_modified = false;                                                              \
		ItemPointerCopy(&tuple->t_self, &(entry)->tid);                                            \
                                                                                                   \
		if (should_free)                                                                           \
			heap_freetuple(tuple);                                                                 \
	} while (0);

static void
invalidation_entry_set_from_hyper_invalidation(Invalidation *entry, const TupleInfo *ti,
											   int32 hyper_id, Oid dimtype,
											   const ContinuousAggBucketFunction *bucket_function)
{
	INVALIDATION_ENTRY_SET(entry,
						   ti,
						   hypertable_id,
						   Form_continuous_aggs_hypertable_invalidation_log);
	/* Since hypertable invalidations are moved to the continuous aggregate
	 * invalidation log, a different hypertable ID must be set (the ID of the
	 * materialized hypertable). */
	entry->hyper_id = hyper_id;
	invalidation_expand_to_bucket_boundaries(entry, dimtype, bucket_function);
}

static void
invalidation_entry_set_from_cagg_invalidation(Invalidation *entry, const TupleInfo *ti, Oid dimtype,
											  const ContinuousAggBucketFunction *bucket_function)
{
	INVALIDATION_ENTRY_SET(entry,
						   ti,
						   materialization_id,
						   Form_continuous_aggs_materialization_invalidation_log);

	/* It isn't strictly necessary to expand the invalidation to bucket
	 * boundaries here since all invalidations were already expanded when
	 * copied from the hypertable invalidation log. However, since
	 * invalidation expansion wasn't implemented in early 2.0.x versions of
	 * the extension, there might be unexpanded entries in the cagg
	 * invalidation log for some users. Therefore we try to expand
	 * invalidation entries also here, although in most cases it would do
	 * nothing. */
	invalidation_expand_to_bucket_boundaries(entry, dimtype, bucket_function);
}

/*
 * Check if two invalidations can be merged into one.
 *
 * Since invalidations are inclusive in both ends, two adjacent invalidations
 * can be merged.
 */
static bool
invalidations_can_be_merged(const Invalidation *a, const Invalidation *b)
{
	/* To account for adjacency, expand one window 1 step in each
	 * direction. This makes adjacent invalidations overlapping. */
	int64 a_start = int64_saturating_sub(a->lowest_modified_value, 1);
	int64 a_end = int64_saturating_add(a->greatest_modified_value, 1);

	return a_end >= b->lowest_modified_value && a_start <= b->greatest_modified_value;
}

/*
 * Try to merge two invalidations into one.
 *
 * Returns true if the invalidations were merged, otherwise false.
 *
 * Given that we scan ordered on lowest_modified_value, the previous and
 * current invalidation can overlap in two ways (generalized):
 *
 * |------|
 *    |++++++++|
 *
 * |-------------|
 *    |++++++++|
 *
 * The closest non-overlapping case is (note that adjacent invalidations can
 * be merged since they are inclusive in both ends):
 *
 * |--|
 *     |++++++++|
 *
 */
static bool
invalidation_entry_try_merge(Invalidation *entry, const Invalidation *newentry)
{
	if (!IsValidInvalidation(newentry))
	{
		return false;
	}

	/* Quick exit if no overlap */
	if (!invalidations_can_be_merged(entry, newentry))
	{
		return false;
	}

	/* Check if the new entry expands beyond the old one (first case above) */
	if (entry->greatest_modified_value < newentry->greatest_modified_value)
	{
		entry->greatest_modified_value = newentry->greatest_modified_value;
		entry->is_modified = true;
	}

	return true;
}

static void
insert_new_cagg_invalidation(const HypertableInvalidationState *state, const Invalidation *entry,
							 int32 cagg_hyper_id)
{
	CatalogSecurityContext sec_ctx;
	TupleDesc tupdesc = RelationGetDescr(state->cagg_log_rel);
	HeapTuple tuple = create_invalidation_tup(tupdesc,
											  cagg_hyper_id,
											  entry->lowest_modified_value,
											  entry->greatest_modified_value);

	ts_catalog_database_info_become_owner(ts_catalog_database_info_get(), &sec_ctx);
	ts_catalog_insert_only(state->cagg_log_rel, tuple);
	ts_catalog_restore_user(&sec_ctx);
	heap_freetuple(tuple);
}

/*
 * Process invalidations in the hypertable invalidation log.
 *
 * Copy and delete all entries from the hypertable invalidation log.
 *
 * Note that each entry gets one copy per continuous aggregate in the cagg
 * invalidation log (unless it was merged or matched the refresh
 * window). These copied entries are later used to track invalidations across
 * refreshes on a per-cagg basis.
 *
 * After this function has run, there are no entries left in the hypertable
 * invalidation log.
 */
static void
move_invalidations_from_hyper_to_cagg_log(const HypertableInvalidationState *state)
{
	const ContinuousAggInfo *all_caggs = state->all_caggs;
	int32 hyper_id = state->hypertable_id;
	int32 last_cagg_hyper_id;
	ListCell *lc1, *lc2;

	last_cagg_hyper_id = llast_int(all_caggs->mat_hypertable_ids);

	/* We use a per-tuple memory context in the scan loop since we could be
	 * processing a lot of invalidations (basically an unbounded
	 * amount). Initialize it here by resetting it. */
	MemoryContextReset(state->per_tuple_mctx);

	/*
	 * Looping over all continuous aggregates in the outer loop ensures all
	 * tuples for a specific continuous aggregate is inserted consecutively in
	 * the cagg invalidation log. This creates better locality for scanning
	 * the invalidations later.
	 */
	forboth (lc1, all_caggs->mat_hypertable_ids, lc2, all_caggs->bucket_functions)
	{
		int32 cagg_hyper_id = lfirst_int(lc1);
		const ContinuousAggBucketFunction *bucket_function = lfirst(lc2);

		Invalidation mergedentry;
		ScanIterator iterator;

		invalidation_entry_reset(&mergedentry);
		hypertable_invalidation_scan_init(&iterator, hyper_id, RowExclusiveLock);
		iterator.ctx.snapshot = state->snapshot;

		/* Scan all invalidations */
		ts_scanner_foreach(&iterator)
		{
			TupleInfo *ti;
			MemoryContext oldmctx;
			Invalidation logentry;

			oldmctx = MemoryContextSwitchTo(state->per_tuple_mctx);
			ti = ts_scan_iterator_tuple_info(&iterator);

			invalidation_entry_set_from_hyper_invalidation(&logentry,
														   ti,
														   cagg_hyper_id,
														   state->dimtype,
														   bucket_function);

			if (!IsValidInvalidation(&mergedentry))
			{
				mergedentry = logentry;
				mergedentry.hyper_id = cagg_hyper_id;
			}
			else if (!invalidation_entry_try_merge(&mergedentry, &logentry))
			{
				insert_new_cagg_invalidation(state, &mergedentry, cagg_hyper_id);
				mergedentry = logentry;
			}

			if (cagg_hyper_id == last_cagg_hyper_id)
			{
				CatalogSecurityContext sec_ctx;

				/* The invalidation has been processed for all caggs, so the
				 * only thing left is to delete it from the source hypertable
				 * invalidation log. */
				ts_catalog_database_info_become_owner(ts_catalog_database_info_get(), &sec_ctx);
				ts_catalog_delete_tid_only(ti->scanrel, &logentry.tid);
				ts_catalog_restore_user(&sec_ctx);
			}

			MemoryContextSwitchTo(oldmctx);
			MemoryContextReset(state->per_tuple_mctx);
		}

		ts_scan_iterator_close(&iterator);

		/* Handle the last merged invalidation */
		if (IsValidInvalidation(&mergedentry))
		{
			insert_new_cagg_invalidation(state, &mergedentry, cagg_hyper_id);
		}
	}
}

static void
cagg_invalidations_scan_by_hypertable_init(ScanIterator *iterator, int32 cagg_hyper_id,
										   LOCKMODE lockmode, int64 window_end)
{
	*iterator = ts_scan_iterator_create(CONTINUOUS_AGGS_MATERIALIZATION_INVALIDATION_LOG,
										lockmode,
										CurrentMemoryContext);
	iterator->ctx.index = catalog_get_index(ts_catalog_get(),
											CONTINUOUS_AGGS_MATERIALIZATION_INVALIDATION_LOG,
											CONTINUOUS_AGGS_MATERIALIZATION_INVALIDATION_LOG_IDX);
	ts_scan_iterator_scan_key_init(
		iterator,
		Anum_continuous_aggs_materialization_invalidation_log_idx_materialization_id,
		BTEqualStrategyNumber,
		F_INT4EQ,
		Int32GetDatum(cagg_hyper_id));
	ts_scan_iterator_scan_key_init(
		iterator,
		Anum_continuous_aggs_materialization_invalidation_log_idx_lowest_modified_value,
		BTLessStrategyNumber,
		F_INT8LT,
		Int64GetDatum(window_end));
}

/*
 * Cut an invalidation and return the part, if any, that remains within the
 * refresh window.
 */
static Invalidation
cut_cagg_invalidation(const ContinuousAggInvalidationState *state,
					  const InternalTimeRange *refresh_window, const Invalidation *entry)
{
	InvalidationResult result;
	Invalidation inner_range;
	ItemPointerData tid = entry->tid;

	invalidation_entry_reset(&inner_range);

	result = cut_invalidation_along_refresh_window(state, entry, refresh_window, &inner_range);

	switch (result)
	{
		case INVAL_NOMATCH:
			/* Entries completely outside the refresh window are skipped before
			 * reaching this function, so NOMATCH should never occur here. */
			Assert("NOMATCH should never occur here" == 0);
			break;
		case INVAL_FULLY_MATCH:
			/* Entry is left in the log for Txn 3. If it was merged with an adjacent
			 * entry, update it to reflect the expanded range. */
			if (entry->is_modified)
			{
				write_invalidation_entry(state->cagg_log_rel, entry, &tid);
			}
			break;
		case INVAL_CUT:
			/* Nothing to do, because the cuts have been written back to the invalidation log
			 * and the last merged inner range will be written at the end of
			 * process_cagg_invalidations_for_refresh*/
			break;
	}

	return inner_range;
}

static Invalidation
cut_cagg_invalidation_and_compute_inner_range(const ContinuousAggInvalidationState *state,
											  const InternalTimeRange *refresh_window,
											  const Invalidation *mergedentry,
											  const Invalidation *current_inner_range)
{
	Invalidation new_inner_range;
	Invalidation inner_range = *current_inner_range;

	/* The previous and current invalidation could not be merged. We
	 * need to cut the prev invalidation against the refresh window */
	new_inner_range = cut_cagg_invalidation(state, refresh_window, mergedentry);

	if (!IsValidInvalidation(&inner_range))
	{
		inner_range = new_inner_range;
	}
	else if (IsValidInvalidation(&new_inner_range) &&
			 !invalidation_entry_try_merge(&inner_range, &new_inner_range))
	{
		insert_invalidation_entry(state->cagg_log_rel, &inner_range);
		inner_range = new_inner_range;
	}

	return inner_range;
}

/*
 * Process cagg invalidation log entries against a refresh window (Txn 2).
 *
 * Scans the cagg invalidation log for entries that overlap the refresh window
 * and splits them so that the parts inside and outside the window become
 * separate log entries. Entries completely before the window are skipped.
 *
 * For each overlapping entry:
 *   - INVAL_FULLY_MATCH: entry is completely enclosed by the window and left
 *     in the log as-is for Txn 3 to materialize (updated if it was merged
 *     with an adjacent entry).
 *   - INVAL_CUT: entry partially overlaps the window and is split. The outside
 *     part(s) are written back to the log; the inside part is inserted as a
 *     new log entry for Txn 3.
 *
 * Because the refresh window has been inscribed to bucket boundaries by the
 * caller, all resulting inside and outside parts written back to the log are
 * guaranteed to start and end on bucket boundaries.
 *
 * Note that the refresh window is inclusive at the start and exclusive at the
 * end. This function assumes that invalidations are scanned in order of
 * lowest_modified_value.
 */
static void
process_cagg_invalidations_for_refresh(const ContinuousAggInvalidationState *state,
									   const InternalTimeRange *refresh_window)
{
	ScanIterator iterator;
	Invalidation mergedentry;
	Invalidation inner_range;

	invalidation_entry_reset(&mergedentry);
	invalidation_entry_reset(&inner_range);

	cagg_invalidations_scan_by_hypertable_init(&iterator,
											   state->cagg->data.mat_hypertable_id,
											   RowExclusiveLock,
											   refresh_window->end);
	iterator.ctx.data = (void *) &state;
	iterator.ctx.snapshot = state->snapshot;
	ScanTupLock scantuplock = {
		.waitpolicy = LockWaitSkip,
		.lockmode = LockTupleExclusive,
		.lockflags = TUPLE_LOCK_FLAG_FIND_LAST_VERSION,
	};
	iterator.ctx.tuplock = &scantuplock;
	iterator.ctx.flags = SCANNER_F_KEEPLOCK;

	/*
	 * Concurrent refreshes on the same cagg are safe from deadlock because all
	 * sessions scan the same index in ascending lowest_modified_value order,
	 * ensuring exclusive tuple locks are always acquired in a consistent order.
	 * However, we have this problem:
	 * S1:  in txn3 materializing ranges  [ 40, 60 ].
	 *      so it will read rows from mat. inv log.
	 *      delete + insert rows into cagg's mat hypertable
	 *      delete row from mat inv log
	 * S2: in txn2 reading invalidation logs in [ 60 80]
	 *     this will read all the logs starting from the beginning
	 *      and < 80 so that it can process overlaps.
	 * S1: locks row for tuple 45
	 *                              S2 : acquires ShareUpdateExlcusiveLock
	 *                              S2:  waits for lock for tuple 45
	 * S1: needs to create a new chunk in mat hypertable
	 *     blocks on S2's ShareUpdateExclusiveLock
	 *     so we have a deadlock
	 * To prevent that in txn2 don't block on locks for mat inv log rows.
	 * it is being concurrently processed by another session.
	 * use waitpolicy = LockWaitSlip
	 */

	MemoryContextReset(state->per_tuple_mctx);

	/* Process all invalidations for the continuous aggregate */
	ts_scanner_foreach(&iterator)
	{
		TupleInfo *ti = ts_scan_iterator_tuple_info(&iterator);
		MemoryContext oldmctx;
		Invalidation logentry;

		oldmctx = MemoryContextSwitchTo(state->per_tuple_mctx);
		invalidation_entry_set_from_cagg_invalidation(&logentry,
													  ti,
													  state->cagg->partition_type,
													  state->cagg->bucket_function);

		/* Skip entries completely before the refresh window — nothing to do */
		if (logentry.greatest_modified_value < refresh_window->start)
		{
			MemoryContextSwitchTo(oldmctx);
			MemoryContextReset(state->per_tuple_mctx);
			continue;
		}

		if (!IsValidInvalidation(&mergedentry))
		{
			mergedentry = logentry;
		}
		else if (invalidation_entry_try_merge(&mergedentry, &logentry))
		{
			/*
			 * The previous and current invalidation were merged into
			 * one entry (i.e., they overlapped or were adjacent).
			 */
			ts_catalog_delete_tid_only(state->cagg_log_rel, &logentry.tid);
		}
		else
		{
			inner_range = cut_cagg_invalidation_and_compute_inner_range(state,
																		refresh_window,
																		&mergedentry,
																		&inner_range);
			mergedentry = logentry;
		}

		MemoryContextSwitchTo(oldmctx);
		MemoryContextReset(state->per_tuple_mctx);
	}

	ts_scan_iterator_close(&iterator);

	/* Handle the last (merged) invalidation */
	if (IsValidInvalidation(&mergedentry))
	{
		inner_range = cut_cagg_invalidation_and_compute_inner_range(state,
																	refresh_window,
																	&mergedentry,
																	&inner_range);
	}

	/* Write the last (merged) inner range back to the cagg invalidation log */
	insert_invalidation_entry(state->cagg_log_rel, &inner_range);
}

static void
cagg_invalidation_state_init(ContinuousAggInvalidationState *state, const ContinuousAgg *cagg)
{
	state->cagg = cagg;
	state->cagg_log_rel = open_cagg_table(CAGG_INVALIDATION_LOG, RowExclusiveLock);
	state->per_tuple_mctx = AllocSetContextCreate(CurrentMemoryContext,
												  "Materialization invalidations",
												  ALLOCSET_DEFAULT_SIZES);
	state->snapshot = RegisterSnapshot(GetTransactionSnapshot());
}

static void
cagg_invalidation_state_cleanup(const ContinuousAggInvalidationState *state)
{
	table_close(state->cagg_log_rel, NoLock);
	UnregisterSnapshot(state->snapshot);
	MemoryContextDelete(state->per_tuple_mctx);
}

static void
hypertable_invalidation_state_init(HypertableInvalidationState *state, int32 hypertable_id,
								   Oid dimtype, const ContinuousAggInfo *all_caggs)
{
	state->hypertable_id = hypertable_id;
	state->dimtype = dimtype;
	state->all_caggs = all_caggs;
	state->cagg_log_rel = open_cagg_table(CAGG_INVALIDATION_LOG, RowExclusiveLock);
	state->per_tuple_mctx = AllocSetContextCreate(CurrentMemoryContext,
												  "Hypertable invalidations",
												  ALLOCSET_DEFAULT_SIZES);
	state->snapshot = RegisterSnapshot(GetTransactionSnapshot());
}

static void
hypertable_invalidation_state_cleanup(const HypertableInvalidationState *state)
{
	table_close(state->cagg_log_rel, NoLock);
	UnregisterSnapshot(state->snapshot);
	MemoryContextDelete(state->per_tuple_mctx);
}

/*
 * Move invalidations for a single hypertable from hypertable invalidation log
 * to materialization invalidation log. This will move *all* hypertable
 * invalidations for the hypertable to the associated continuous aggregates.
 */
void
invalidation_process_hypertable_log(int32 hypertable_id, Oid dimtype)
{
	HypertableInvalidationState state;
	const ContinuousAggInfo all_caggs = ts_continuous_agg_get_all_caggs_info(hypertable_id);

	hypertable_invalidation_state_init(&state, hypertable_id, dimtype, &all_caggs);
	move_invalidations_from_hyper_to_cagg_log(&state);
	hypertable_invalidation_state_cleanup(&state);
}

void
invalidation_process_cagg_log(const ContinuousAgg *cagg, const InternalTimeRange *refresh_window)
{
	ContinuousAggInvalidationState state;

	cagg_invalidation_state_init(&state, cagg);
	process_cagg_invalidations_for_refresh(&state, refresh_window);
	cagg_invalidation_state_cleanup(&state);
}

/*
 * Scanner filter that excludes cagg invalidation log entries whose
 * greatest_modified_value extends beyond the refresh window end.
 * The exclusive window_end (int64 *) is passed as data.
 */
static ScanFilterResult
cagg_invalidation_window_end_filter(const TupleInfo *ti, void *data)
{
	int64 window_end = *(const int64 *) data;
	bool isnull;
	Datum greatest =
		slot_getattr(ti->slot,
					 Anum_continuous_aggs_materialization_invalidation_log_greatest_modified_value,
					 &isnull);

	Assert(!isnull);
	return DatumGetInt64(greatest) < window_end ? SCAN_INCLUDE : SCAN_EXCLUDE;
}

/*
 * Read all cagg invalidation log entries that fall entirely within
 * [refresh_window->start, refresh_window->end) — i.e., entries written by
 * Txn 2 as "ready to materialize" — collect them into an InvalidationStore,
 * and delete them from the log.
 *
 * Txn 2 cuts all log entries to be either fully inside or fully outside the
 * refresh window.  However, between Txn 2 and Txn 3, a concurrent session's
 * Txn 1 (invalidation_process_hypertable_log) may insert new cagg log entries
 * that partially overlap the window and have not been cut.  The scan
 * skips those entries.
 *
 * The (materialization_id, lowest_modified_value) index is used for a range
 * scan scoped to [window_start, window_end).  The greatest_modified_value
 * check is applied as a heap-side filter via the scanner's filter callback.
 *
 * Returns an InvalidationStore with the collected entries, or NULL if no
 * entries were found.
 *
 * NOTE: Txn2 can process only entries strictly falling within its refresh window.
 * This is because a concurrent transaction could be processing an adjacent ranges.
 * So Txn2 can no longer attempt to work on entries outside its refresh window.
 * When this transaction commits, all new entries written by it will be visible to
 * refresh processes in Txn 3.
 *
 */
InvalidationStore *
collect_and_delete_cagg_invalidations_in_window(const ContinuousAgg *cagg,
												const InternalTimeRange *refresh_window, bool force)
{
	ScanIterator iterator;
	int64 window_end = refresh_window->end;
	InvalidationStore *store = NULL;
	TupleDesc tupdesc = NULL;
	Tuplestorestate *tupstore = NULL;

	if (force)
	{
		/*
		 * Forced refresh: pre-build a store covering the entire window.
		 * Log entries will be deleted below but not collected into the store.
		 */
		Relation rel = open_cagg_table(CAGG_INVALIDATION_LOG, AccessShareLock);
		tupdesc = CreateTupleDescCopy(RelationGetDescr(rel));
		table_close(rel, AccessShareLock);

		int64 inclusive_end = ts_time_saturating_sub(refresh_window->end, 1, refresh_window->type);
		HeapTuple forced_tuple = create_invalidation_tup(tupdesc,
														 cagg->data.mat_hypertable_id,
														 refresh_window->start,
														 inclusive_end);
		tupstore = tuplestore_begin_heap(false, false, work_mem);
		tuplestore_puttuple(tupstore, forced_tuple);
		heap_freetuple(forced_tuple);
	}

	cagg_invalidations_scan_by_hypertable_init(&iterator,
											   cagg->data.mat_hypertable_id,
											   RowExclusiveLock,
											   window_end);
	/* Add lower bound: only entries whose start is inside the refresh window */
	ts_scan_iterator_scan_key_init(
		&iterator,
		Anum_continuous_aggs_materialization_invalidation_log_idx_lowest_modified_value,
		BTGreaterEqualStrategyNumber,
		F_INT8GE,
		Int64GetDatum(refresh_window->start));
	/* Filter out entries that extend beyond the window end */
	iterator.ctx.filter = cagg_invalidation_window_end_filter;
	iterator.ctx.data = &window_end;

	ts_scanner_foreach(&iterator)
	{
		TupleInfo *ti = ts_scan_iterator_tuple_info(&iterator);

		if (!force)
		{
			if (tupdesc == NULL)
			{
				/* Initialize the tuplestore on first match */
				tupstore = tuplestore_begin_heap(false, false, work_mem);
				tupdesc = CreateTupleDescCopy(RelationGetDescr(ti->scanrel));
			}

			bool should_free;
			HeapTuple tuple = ts_scanner_fetch_heap_tuple(ti, false, &should_free);
			tuplestore_puttuple(tupstore, tuple);
			if (should_free)
			{
				heap_freetuple(tuple);
			}
		}

		ts_catalog_delete_tid_only(ti->scanrel, ts_scanner_get_tuple_tid(ti));
	}
	ts_scan_iterator_close(&iterator);

	if (tupstore != NULL)
	{
		store = palloc(sizeof(InvalidationStore));
		store->tupstore = tupstore;
		store->tupdesc = tupdesc;
	}

	return store;
}

void
invalidation_store_free(InvalidationStore *store)
{
	FreeTupleDesc(store->tupdesc);
	tuplestore_end(store->tupstore);
	pfree(store);
}

bool
invalidation_hypertable_has_invalidations(int32 hyper_id)
{
	bool found = false;
	ScanIterator iterator;
	hypertable_invalidation_scan_init(&iterator, hyper_id, AccessShareLock);
	iterator.ctx.limit = 1; /* we only need to know if there is at least one */

	ts_scan_iterator_start_scan(&iterator);
	if (ts_scan_iterator_next(&iterator))
	{
		found = true;
	}
	ts_scan_iterator_close(&iterator);

	return found;
}

bool
invalidation_cagg_has_pending_mat_ranges(ContinuousAgg *cagg)
{
	bool found = false;
	int32 cagg_hyper_id = cagg->data.mat_hypertable_id;

	ScanIterator iterator = ts_scan_iterator_create(CONTINUOUS_AGGS_MATERIALIZATION_RANGES,
													AccessShareLock,
													CurrentMemoryContext);
	iterator.ctx.index = catalog_get_index(ts_catalog_get(),
										   CONTINUOUS_AGGS_MATERIALIZATION_RANGES,
										   CONTINUOUS_AGGS_MATERIALIZATION_RANGES_IDX);
	ts_scan_iterator_scan_key_init(&iterator,
								   Anum_continuous_aggs_materialization_ranges_materialization_id,
								   BTEqualStrategyNumber,
								   F_INT4EQ,
								   Int32GetDatum(cagg_hyper_id));
	iterator.ctx.limit = 1; /* we only need to know if there is at least one */

	ts_scan_iterator_start_scan(&iterator);
	if (ts_scan_iterator_next(&iterator))
	{
		found = true;
	}
	ts_scan_iterator_close(&iterator);

	return found;
}

bool
invalidation_cagg_has_invalidations(ContinuousAgg *cagg)
{
	bool found = false;
	int32 cagg_hyper_id = cagg->data.mat_hypertable_id;

	ScanIterator iterator;
	cagg_invalidations_scan_by_hypertable_init(&iterator,
											   cagg_hyper_id,
											   AccessShareLock,
											   PG_INT64_MAX);

	int64 watermark = ts_cagg_watermark_get(cagg_hyper_id);
	ts_scanner_foreach(&iterator)
	{
		TupleInfo *ti;
		Invalidation logentry;

		ti = ts_scan_iterator_tuple_info(&iterator);

		invalidation_entry_set_from_cagg_invalidation(&logentry,
													  ti,
													  cagg->partition_type,
													  cagg->bucket_function);
		/* Entries which cannot be invalidations */
		if (logentry.greatest_modified_value == INVAL_NEG_INFINITY ||
			logentry.lowest_modified_value >= watermark)
		{
			continue;
		}
		else
		{
			found = true;
			break;
		}
	}
	ts_scan_iterator_close(&iterator);

	return found;
}
