Decode binary local timestamps like PostgreSQL around DST changes (0 mismatches over 2.45M values, 1 object instead of 2)

.github/workflows/binary-gems.yml

@@ -74,7 +74,7 @@ jobs:
           - os: ubuntu-latest
             ruby: "4.0"
             platform: "x86_64-linux"
-          - os: ubuntu-latest
+          - os: ubuntu-24.04
             ruby: "3.1"
             platform: "x86_64-linux"
           - os: ubuntu-24.04-arm

ext/pg_binary_decoder.c

@@ -284,6 +284,82 @@ pg_bin_dec_array(t_pg_coder *conv, const char *input_line, int len, int tuple, i
 
 #define PG_INT64_MIN	(-0x7FFFFFFFFFFFFFFFL - 1)
 #define PG_INT64_MAX	0x7FFFFFFFFFFFFFFFL
+#define SECS_PER_DAY	86400
+
+/*
+ * Get the UTC offset of the local time zone at the given instant.
+ * Returns 0 when the C library can't convert it.
+ */
+static int
+local_utc_offset(int64_t seconds, int *offset)
+{
+	time_t time = (time_t)seconds;
+	int64_t utc_days = seconds / SECS_PER_DAY - (seconds % SECS_PER_DAY < 0);
+	int utc_weekday = (int)(((utc_days + 4) % 7 + 7) % 7); /* 1970-01-01 was a Thursday */
+	int local_days_ahead;
+	struct tm tm;
+
+	if( time != seconds || localtime_r(&time, &tm) == NULL ) return 0;
+	/* local and UTC dates are at most a day apart */
+	local_days_ahead = (tm.tm_wday - utc_weekday + 7) % 7;
+	if( local_days_ahead == 6 ) local_days_ahead = -1;
+	*offset = local_days_ahead * SECS_PER_DAY + tm.tm_hour * 3600 + tm.tm_min * 60 + tm.tm_sec
+			- (int)(seconds - utc_days * SECS_PER_DAY);
+	return 1;
+}
+
+/*
+ * Get the UTC offset to interpret a wall clock as local time, given as seconds since 1970-01-01 00:00:00.
+ * Like PostgreSQL's DetermineTimeZoneOffset(), a wall clock in a DST gap or overlap takes the later instant.
+ * Returns 0 when the C library can't convert it.
+ */
+static int
+wall_clock_utc_offset(int64_t wall_clock, int *offset)
+{
+	int offset_before, offset_after, offset_at_before, offset_at_after;
+
+	/* Before 1901-12-14, Ruby on macOS renders local times without the LMT offsets of the C library */
+	if( wall_clock < (int64_t)INT32_MIN + SECS_PER_DAY ) return 0;
+
+#ifdef _WIN32
+	_tzset();
+#else
+	tzset();
+#endif
+	if( !local_utc_offset(wall_clock - SECS_PER_DAY, &offset_before) ||
+			!local_utc_offset(wall_clock + SECS_PER_DAY, &offset_after) ) return 0;
+
+	if( offset_before == offset_after ){
+		*offset = offset_before;
+		return 1;
+	}
+
+	/* A transition within a day: keep the only interpretation that gives back the wall clock, if any */
+	if( !local_utc_offset(wall_clock - offset_before, &offset_at_before) ||
+			!local_utc_offset(wall_clock - offset_after, &offset_at_after) ) return 0;
+	if( (offset_at_before == offset_before) != (offset_at_after == offset_after) ){
+		*offset = offset_at_before == offset_before ? offset_before : offset_after;
+	} else {
+		*offset = offset_before < offset_after ? offset_before : offset_after;
+	}
+	return 1;
+}
+
+/*
+ * Interpret the wall clock with the UTC offset Ruby gives to it read as UTC, when the C library can't convert it.
+ * That's off by the DST shift within the offset's distance of a DST change, which doesn't happen before 1901.
+ */
+static VALUE
+time_local_from_wall_clock(t_pg_coder *conv, int64_t sec, int64_t nsec)
+{
+	VALUE t = rb_funcall(rb_cTime, rb_intern("at"), 3, LL2NUM(sec), LL2NUM(nsec), ID2SYM(rb_intern("nsec")));
+
+	t = rb_funcall(t, rb_intern("-"), 1, rb_funcall(t, rb_intern("utc_offset"), 0));
+	if( !(conv->flags & PG_CODER_TIMESTAMP_APP_LOCAL) ) {
+		t = rb_funcall(t, rb_intern("getutc"), 0);
+	}
+	return t;
+}
 
 /*
  * Document-class: PG::BinaryDecoder::Timestamp < PG::SimpleDecoder
@@ -326,6 +402,16 @@ pg_bin_dec_timestamp(t_pg_coder *conv, const char *val, int len, int tuple, int
 			sec = (timestamp / 1000000) + 10957L * 24L * 3600L;
 			nsec = (timestamp % 1000000) * 1000;
 
+			if( conv->flags & PG_CODER_TIMESTAMP_DB_LOCAL ) {
+				/* interpret it as local time */
+				int offset;
+
+				if( !wall_clock_utc_offset(sec - (nsec < 0), &offset) ){
+					return time_local_from_wall_clock(conv, sec, nsec);
+				}
+				sec -= offset;
+			}
+
 #if defined(NEGATIVE_TIME_T) && SIZEOF_TIME_T >= 8
 			/* Fast path for time conversion */
 			{
@@ -338,10 +424,6 @@ pg_bin_dec_timestamp(t_pg_coder *conv, const char *val, int len, int tuple, int
 				t = rb_funcall(t, rb_intern("utc"), 0);
 			}
 #endif
-			if( conv->flags & PG_CODER_TIMESTAMP_DB_LOCAL ) {
-				/* interpret it as local time */
-				t = rb_funcall(t, rb_intern("-"), 1, rb_funcall(t, rb_intern("utc_offset"), 0));
-			}
 			return t;
 	}
 }

ext/pg_text_decoder.c

@@ -747,6 +747,10 @@ static VALUE pg_text_dec_timestamp(t_pg_coder *conv, const char *val, int len, i
 				tm.tm_sec = sec;
 				tm.tm_isdst = -1;
 
+#ifdef _WIN32
+				/* Windows' mktime() reads TZ only once, unlike POSIX: pick up a TZ changed since */
+				_tzset();
+#endif
 				ts.tv_sec = mktime(&tm);
 				if (ts.tv_sec != -1) {
 					return rb_time_timespec_new(&ts, conv->flags & PG_CODER_TIMESTAMP_APP_LOCAL ? INT_MAX : INT_MAX-1);

spec/pg/type_spec.rb

@@ -250,6 +250,46 @@ def textdec_timestamptz_decode_should_fail(str)
 				end
 			end
 
+			context 'binary local timestamps' do
+				around do |example|
+					tz = ENV['TZ']
+					# Windows compatible form of 'Europe/Paris'
+					ENV['TZ'] = 'CET-1CEST,M3.5.0,M10.5.0/3'
+					example.run
+				ensure
+					ENV['TZ'] = tz
+				end
+				before do
+					skip "Europe/Paris isn't available as TZ" unless Time.local(2024, 7, 1).utc_offset == 7200
+				end
+
+				# Instants given by PostgreSQL for `wall_clock::timestamp AT TIME ZONE 'Europe/Paris'`
+				{
+					[2024, 1, 15, 12, 0] => Time.utc(2024, 1, 15, 11, 0),
+					[2024, 7, 1, 12, 0, Rational(123456, 1000000)] => Time.utc(2024, 7, 1, 10, 0, Rational(123456, 1000000)),
+					[2024, 3, 31, 1, 30] => Time.utc(2024, 3, 31, 0, 30),
+					[2024, 3, 31, 2, 30] => Time.utc(2024, 3, 31, 1, 30), # DST gap
+					[2024, 3, 31, 3, 30] => Time.utc(2024, 3, 31, 1, 30),
+					[2024, 10, 27, 1, 30] => Time.utc(2024, 10, 26, 23, 30),
+					[2024, 10, 27, 2, 30] => Time.utc(2024, 10, 27, 1, 30), # DST overlap
+					[2024, 10, 27, 3, 30] => Time.utc(2024, 10, 27, 2, 30),
+					[2038, 10, 31, 2, 30] => Time.utc(2038, 10, 31, 1, 30), # DST overlap beyond 32 bit time_t
+					[2040, 3, 25, 2, 30] => Time.utc(2040, 3, 25, 1, 30), # DST gap beyond 32 bit time_t
+				}.each do |wall_clock, instant|
+					it "decodes #{wall_clock.first(5)} as local time like PostgreSQL" do
+						wire = binaryenc_timestamputc.encode(Time.utc(*wall_clock))
+
+						time = PG::BinaryDecoder::TimestampLocal.new.decode(wire)
+						expect( time ).to eq( instant )
+						expect( time.utc? ).to be false
+
+						time = PG::BinaryDecoder::Timestamp.new(flags: PG::Coder::TIMESTAMP_DB_LOCAL).decode(wire)
+						expect( time ).to eq( instant )
+						expect( time.utc? ).to be true
+					end
+				end
+			end
+
 			context 'dates' do
 				let(:textdec_date) { PG::TextDecoder::Date.new }