🔍 s&box Package Code Search
Search C# source code, UI razor templates, shaders, and configs across s&box packages.
🔗 Raw Facepunch API Link: https://public.facepunch.com/sbox/code/search/1/?ident=jeffskitchen.llm_poc&take=20
Showing code results for query:
*
(60 total matches found)
Game
game
VectorControl
{
gap: 2px;
flex-grow: 1;
}
VectorControl NumberEntry
{
flex-basis: 50%;
}
Game
game
ControlSheetGroupHeader
{
font-size: 1.33rem;
color: red;
gap: 2px;
align-items: center;
&.hidden
{
display: none;
}
> .title
{
font-weight: 600;
}
&.has-toggle
{
cursor: pointer;
opacity: 0.8;
&:before
{
content: ' ';
width: 22px;
height: 22px;
background-color: #000a;
align-items: center;
justify-content: center;
text-align: center;
border-radius: 5px;
border: 1px solid #555;
}
&:hover
{
opacity: 1;
&:before
{
border-color: #888;
}
}
&.checked
{
> .title
{
color: white;
}
&:before
{
content: '✓';
font-weight: bold;
color: #08f;
border-color: #08f;
}
}
}
}
Game
game
@import "/styles/_theme.scss";
$primary: $primary-blue !default;
$primary-alt: white !default;
$form-control-height: 24px !default;
.checkbox
{
cursor: pointer;
color: rgba( $primary-alt, 0.6 );
align-items: center;
gap: 8px;
label
{
pointer-events: none;
}
> .checkmark
{
padding: 1px;
font-size: 22px;
border: 1px solid $primary;
border-radius: $rounding-small;
text-align: center;
justify-content: center;
align-items: center;
color: transparent;
min-height: $form-control-height;
pointer-events: none;
flex-shrink: 0;
}
&.checked
{
> .checkmark
{
color: $primary-alt;
background-color: $primary;
}
}
&:active
{
color: $primary-alt;
}
&:hover
{
color: $primary-alt;
}
}
Game
game
using System.Text.Json.Serialization;
namespace LlmPoc.Llm;
public sealed class GreedyGenerationReferenceDocument
{
[JsonPropertyName( "model_id" )]
public string ModelId { get; set; }
[JsonPropertyName( "prompt" )]
public string Prompt { get; set; }
[JsonPropertyName( "input_token_ids" )]
public int[] InputTokenIds { get; set; }
[JsonPropertyName( "bos_token_id" )]
public int BosTokenId { get; set; }
[JsonPropertyName( "eos_token_id" )]
public int EosTokenId { get; set; }
[JsonPropertyName( "bos_automatically_added" )]
public bool BosAutomaticallyAdded { get; set; }
[JsonPropertyName( "eos_automatically_added" )]
public bool EosAutomaticallyAdded { get; set; }
[JsonPropertyName( "do_sample" )]
public bool DoSample { get; set; }
[JsonPropertyName( "argmax_tie_break" )]
public string ArgmaxTieBreak { get; set; }
[JsonPropertyName( "logits_processors" )]
public string[] LogitsProcessors { get; set; }
[JsonPropertyName( "raw_logits_used_for_argmax" )]
public bool RawLogitsUsedForArgmax { get; set; }
[JsonPropertyName( "cached_and_uncached_sequences_match" )]
public bool CachedAndUncachedSequencesMatch { get; set; }
[JsonPropertyName( "csharp_baseline_use_cache" )]
public bool CsharpBaselineUseCache { get; set; }
[JsonPropertyName( "max_new_tokens" )]
public int MaxNewTokens { get; set; }
[JsonPropertyName( "generated_token_ids" )]
public int[] GeneratedTokenIds { get; set; }
[JsonPropertyName( "generated_text" )]
public string GeneratedText { get; set; }
[JsonPropertyName( "full_sequence_token_ids" )]
public int[] FullSequenceTokenIds { get; set; }
[JsonPropertyName( "stop_reason" )]
public string StopReason { get; set; }
[JsonPropertyName( "eos_reached" )]
public bool EosReached { get; set; }
[JsonPropertyName( "minimum_top1_top2_margin" )]
public float MinimumTop1Top2Margin { get; set; }
[JsonPropertyName( "minimum_margin_step" )]
public int MinimumMarginStep { get; set; }
[JsonPropertyName( "steps" )]
public GreedyGenerationReferenceStep[] Steps { get; set; }
[JsonPropertyName( "later_full_logit_reference" )]
public GreedyGenerationLogitReference LaterFullLogitReference { get; set; }
}
public sealed class GreedyGenerationReferenceStep
{
[JsonPropertyName( "step" )]
public int Step { get; set; }
[JsonPropertyName( "input_sequence_length" )]
public int InputSequenceLength { get; set; }
[JsonPropertyName( "input_token_ids" )]
public int[] InputTokenIds { get; set; }
[JsonPropertyName( "new_token_position" )]
public int NewTokenPosition { get; set; }
[JsonPropertyName( "expected_next_token_id" )]
public int ExpectedNextTokenId { get; set; }
[JsonPropertyName( "decoded_token" )]
public string DecodedToken { get; set; }
[JsonPropertyName( "top1_logit" )]
public float Top1Logit { get; set; }
[JsonPropertyName( "top2_token_id" )]
public int Top2TokenId { get; set; }
[JsonPropertyName( "top2_logit" )]
public float Top2Logit { get; set; }
[JsonPropertyName( "top1_top2_margin" )]
public float Top1Top2Margin { get; set; }
[JsonPropertyName( "eos_reached" )]
public bool EosReached { get; set; }
[JsonPropertyName( "top5" )]
public ForwardReferenceTopLogit[] TopFive { get; set; }
}
public sealed class GreedyGenerationLogitReference
{
[JsonPropertyName( "step" )]
public int Step { get; set; }
[JsonPropertyName( "input_sequence_length" )]
public int InputSequenceLength { get; set; }
[JsonPropertyName( "file" )]
public string File { get; set; }
[JsonPropertyName( "elements" )]
public int Elements { get; set; }
[JsonPropertyName( "bytes" )]
public int Bytes { get; set; }
[JsonPropertyName( "sha256" )]
public string Sha256 { get; set; }
}
Game
game
namespace LlmPoc.Llm;
public static class LlmPaths
{
public const string RuntimeModelResource = "models/tinystories.llmmdl";
public const string Root = "models/tinystories-instruct-1m";
public const string Model = Root + "/model.bin";
public const string Config = Root + "/config.json";
public const string Manifest = Root + "/tensor_manifest.json";
public const string Reference = Root + "/reference.json";
public const string ReferenceLogits = Root + "/reference_logits.f32";
public const string ReferenceIntermediates = Root + "/reference_intermediates.json";
public const string ReferenceEmbedding = Root + "/reference_embedding.f32";
public const string ReferenceLayer0Ln1 = Root + "/reference_layer0_ln1.f32";
public const string ReferenceLayer0Q = Root + "/reference_layer0_q.f32";
public const string ReferenceLayer0K = Root + "/reference_layer0_k.f32";
public const string ReferenceLayer0V = Root + "/reference_layer0_v.f32";
public const string ReferenceLayer0QHeads = Root + "/reference_layer0_q_heads.f32";
public const string ReferenceLayer0KHeads = Root + "/reference_layer0_k_heads.f32";
public const string ReferenceLayer0VHeads = Root + "/reference_layer0_v_heads.f32";
public const string ReferenceLayer0ScoresScaledUnmasked =
Root + "/reference_layer0_scores_scaled_unmasked.f32";
public const string ReferenceLayer0AttentionMask =
Root + "/reference_layer0_attention_mask.json";
public const string ReferenceLayer0ScoresMaskedPreSoftmax =
Root + "/reference_layer0_scores_masked_pre_softmax.f32";
public const string ReferenceLayer0AttentionProbs =
Root + "/reference_layer0_attention_probs.f32";
public const string ReferenceLayer0AttentionContextHeads =
Root + "/reference_layer0_attention_context_heads.f32";
public const string ReferenceLayer0AttentionMerged =
Root + "/reference_layer0_attention_merged.f32";
public const string ReferenceLayer0AttentionOutProj =
Root + "/reference_layer0_attention_out_proj.f32";
public const string ReferenceLayer0AttentionResidual =
Root + "/reference_layer0_attention_residual.f32";
public const string ReferenceLayer0Ln2 = Root + "/reference_layer0_ln2.f32";
public const string ReferenceLayer0MlpFc = Root + "/reference_layer0_mlp_fc.f32";
public const string ReferenceLayer0MlpGelu = Root + "/reference_layer0_mlp_gelu.f32";
public const string ReferenceLayer0MlpProj = Root + "/reference_layer0_mlp_proj.f32";
public const string ReferenceLayer0Output = Root + "/reference_layer0_output.f32";
public const string ReferenceFinalLayerNorm = Root + "/reference_final_ln.f32";
public const string ReferenceFinalLogitsLastPosition =
Root + "/reference_final_logits_last_position.f32";
public const string ReferenceGreedyGeneration =
Root + "/reference_greedy_generation.json";
public const string ReferenceGenerationStep11Logits =
Root + "/reference_generation_step11_logits.f32";
public const string ReferenceLayer1LocalMaskLen260 =
Root + "/reference_layer1_local_mask_len260.bin";
public const string ReferenceLayer1LocalMaskLen260Metadata =
Root + "/reference_layer1_local_mask_len260.json";
public const string Tokenizer = Root + "/tokenizer/tokenizer.json";
public const string TokenizerConfig = Root + "/tokenizer/tokenizer_config.json";
public static string ReferenceStage( string fileName )
{
if ( string.IsNullOrWhiteSpace( fileName ) || fileName.Contains( '/' ) ||
fileName.Contains( '\\' ) || fileName.Contains( ".." ) )
{
throw new ArgumentException(
$"[LLM:ERROR] Reference stage filename '{fileName}' is invalid.",
nameof( fileName ) );
}
return Root + "/" + fileName;
}
}
Game
game
global using static Sandbox.Internal.GlobalGameNamespace;
global using Microsoft.AspNetCore.Components;
global using Microsoft.AspNetCore.Components.Rendering;
[assembly: global::System.Reflection.AssemblyMetadata( "AddonTitle", "LLM POC" )]
[assembly: global::System.Reflection.AssemblyMetadata( "AddonIdent", "llm_poc" )]
[assembly: global::System.Reflection.AssemblyMetadata( "OrgIdent", "jeffskitchen" )]
[assembly: global::System.Reflection.AssemblyMetadata( "Ident", "jeffskitchen.llm_poc" )]
[assembly: global::System.Reflection.AssemblyMetadata( "EngineVersion", "28" )]
[assembly: global::System.Reflection.AssemblyMetadata( "EngineMinorVersion", "1" )]
[assembly: System.Runtime.Versioning.TargetFramework( ".NETCoreApp,Version=v9.0", FrameworkDisplayName = ".NET 9.0" )]
[assembly: global::System.Reflection.AssemblyMetadata( "CompileTime", "2026-08-17T04:57:47.5628694Z" )]
[assembly: global::System.Reflection.AssemblyVersion("0.0.124.0")]
[assembly: global::System.Reflection.AssemblyFileVersion("0.0.124.0")]
Game
game
global using Sandbox; global using System; global using System.Collections.Generic; global using System.Linq; global using System.Threading.Tasks;
Game
game
using Sandbox.Diagnostics;
namespace LlmPoc.Llm;
public sealed class GreedyGenerationParityResult
{
public GreedyGenerationResult Generation { get; init; }
public NumericComparison LaterStepLogitComparison { get; init; }
public int LaterStepLogitIndex { get; init; }
public float MinimumMargin { get; init; }
public int MinimumMarginStep { get; init; }
public double ValidationHarnessMilliseconds { get; init; }
}
public static class GreedyGenerationParity
{
private const double LogitAbsoluteTolerance = 5.0e-5;
private const double LogitRelativeTolerance = 1.0e-5;
private const double TopLogitAbsoluteTolerance = 5.0e-5;
public static GreedyGenerationParityResult Validate(
SboxLlmModel model,
TinyStoriesConfig config,
Gpt2ByteBpeTokenizer tokenizer,
LlmReferenceData historicalReference,
GreedyGenerationReferenceDocument reference )
{
if ( model is null ) throw new ArgumentNullException( nameof( model ) );
if ( config is null ) throw new ArgumentNullException( nameof( config ) );
if ( tokenizer is null ) throw new ArgumentNullException( nameof( tokenizer ) );
if ( historicalReference is null )
throw new ArgumentNullException( nameof( historicalReference ) );
if ( reference is null ) throw new ArgumentNullException( nameof( reference ) );
ValidateReference( config, tokenizer, historicalReference, reference );
float[] laterExpected = ReferenceFloatData.LoadFromMounted(
LlmPaths.ReferenceStage( reference.LaterFullLogitReference.File ),
reference.LaterFullLogitReference.Elements );
NumericComparison laterComparison = null;
FastTimer validationTimer = FastTimer.StartNew();
GreedyGenerationResult generation = TinyStoriesGreedyGenerator.Generate(
model,
config,
tokenizer,
reference.InputTokenIds,
reference.MaxNewTokens,
observation =>
{
GreedyGenerationStepResult actual = observation.Step;
if ( actual.Step < 0 || actual.Step >= reference.Steps.Length )
{
throw new InvalidOperationException(
$"[LLM:ERROR] Generation produced unexpected step {actual.Step}." );
}
GreedyGenerationReferenceStep expected = reference.Steps[actual.Step];
ValidateStepContext( expected, observation.InputTokenIds );
bool tokenPassed = actual.TokenId == expected.ExpectedNextTokenId;
if ( !tokenPassed )
{
LogMismatch( tokenizer, expected, observation );
throw new InvalidOperationException(
$"[LLM:ERROR] Greedy generation first mismatch at step {actual.Step}: " +
$"expected token {expected.ExpectedNextTokenId}, actual {actual.TokenId}. " +
"No mismatching token was appended." );
}
if ( actual.DecodedToken != expected.DecodedToken )
{
throw new InvalidOperationException(
$"[LLM:ERROR] Generation step {actual.Step} token {actual.TokenId} decoded " +
$"as '{EscapeVisible( actual.DecodedToken )}', Python expected " +
$"'{EscapeVisible( expected.DecodedToken )}'." );
}
ValidateTopFive( tokenizer, expected, actual );
if ( actual.EosReached != expected.EosReached )
{
throw new InvalidOperationException(
$"[LLM:ERROR] Generation step {actual.Step} EOS state expected " +
$"{expected.EosReached}, actual {actual.EosReached}." );
}
if ( actual.Step == reference.LaterFullLogitReference.Step )
{
laterComparison = TensorDiagnostics.Compare(
laterExpected,
observation.Forward.Logits.Data,
LogitAbsoluteTolerance,
LogitRelativeTolerance );
TensorSummary summary = TensorDiagnostics.Summarize(
observation.Forward.Logits );
LlmLog.Info(
"PARITY",
$"generation.step{actual.Step}.logits count={summary.ElementCount:N0} " +
$"finite={summary.FiniteCount:N0}/{summary.ElementCount:N0} " +
$"maxAbs={laterComparison.MaximumAbsoluteError:G12} " +
$"meanAbs={laterComparison.MeanAbsoluteError:G12} " +
$"maxRel={laterComparison.MaximumRelativeError:G12} " +
$"worst_vocab={laterComparison.MaximumErrorIndex} " +
$"expected={laterComparison.ExpectedAtMaximumError:G9} " +
$"actual={laterComparison.ActualAtMaximumError:G9} " +
$"absTol={LogitAbsoluteTolerance:G1} relTol={LogitRelativeTolerance:G1} " +
$"{(laterComparison.Passed ? "PASS" : "FAIL")}" );
if ( !laterComparison.Passed )
{
throw new InvalidOperationException(
$"[LLM:ERROR] Later generation step {actual.Step} full-logit parity " +
$"failed: {laterComparison}." );
}
}
LlmLog.Info(
"GEN",
$"step={actual.Step} context={actual.InputSequenceLength} " +
$"position={actual.NewTokenPosition} expected={expected.ExpectedNextTokenId} " +
$"actual={actual.TokenId} token='{EscapeVisible( actual.DecodedToken )}' " +
$"margin={actual.Top1Top2Margin:G9} forward_ms={actual.ForwardMilliseconds:N4} " +
$"lm_head_ms={actual.LmHeadMilliseconds:N4} PASS" );
} );
RequireExactArray(
"generated token IDs", reference.GeneratedTokenIds, generation.GeneratedTokenIds );
RequireExactArray(
"full generated sequence", reference.FullSequenceTokenIds,
generation.FullSequenceTokenIds );
if ( generation.GeneratedText != reference.GeneratedText )
{
throw new InvalidOperationException(
$"[LLM:ERROR] Generated text expected '{EscapeVisible( reference.GeneratedText )}', " +
$"actual '{EscapeVisible( generation.GeneratedText )}'." );
}
if ( generation.StopReason != reference.StopReason ||
generation.EosReached != reference.EosReached )
{
throw new InvalidOperationException(
$"[LLM:ERROR] Generation stop expected reason={reference.StopReason} " +
$"eos={reference.EosReached}, actual reason={generation.StopReason} " +
$"eos={generation.EosReached}." );
}
if ( laterComparison is null )
{
throw new InvalidOperationException(
$"[LLM:ERROR] Later-step full-logit checkpoint " +
$"{reference.LaterFullLogitReference.Step} did not execute." );
}
GreedyGenerationStepResult minimum = generation.Steps[0];
for ( int index = 1; index < generation.Steps.Length; index++ )
{
if ( generation.Steps[index].Top1Top2Margin < minimum.Top1Top2Margin )
{
minimum = generation.Steps[index];
}
}
if ( minimum.Step != reference.MinimumMarginStep ||
Math.Abs( minimum.Top1Top2Margin - reference.MinimumTop1Top2Margin ) >
TopLogitAbsoluteTolerance )
{
throw new InvalidOperationException(
$"[LLM:ERROR] Minimum generation margin expected step=" +
$"{reference.MinimumMarginStep} value={reference.MinimumTop1Top2Margin:G9}, " +
$"actual step={minimum.Step} value={minimum.Top1Top2Margin:G9}." );
}
LlmLog.Info(
"PARITY",
$"greedy_sequence generated={generation.GeneratedTokenIds.Length} " +
$"matched={generation.GeneratedTokenIds.Length} ids=" +
$"[{string.Join( ",", generation.GeneratedTokenIds )}] " +
$"text='{EscapeVisible( generation.GeneratedText )}' stop={generation.StopReason} PASS" );
LlmLog.Info(
"GEN",
$"minimum_margin={minimum.Top1Top2Margin:G9} step={minimum.Step} " +
$"first_token_ms={generation.Steps[0].StepMilliseconds:N4} " +
$"last_token_ms={generation.Steps[^1].StepMilliseconds:N4} " +
$"total_forward_ms={generation.TotalForwardMilliseconds:N4} " +
$"total_generation_ms={generation.TotalGenerationMilliseconds:N4} " +
$"average_ms_per_token=" +
$"{generation.TotalGenerationMilliseconds / generation.Steps.Length:N4} " +
$"tokens_per_second=" +
$"{generation.Steps.Length * 1000.0 / generation.TotalGenerationMilliseconds:N4} " +
"kv_cache=false PASS" );
return new GreedyGenerationParityResult
{
Generation = generation,
LaterStepLogitComparison = laterComparison,
LaterStepLogitIndex = reference.LaterFullLogitReference.Step,
MinimumMargin = minimum.Top1Top2Margin,
MinimumMarginStep = minimum.Step,
ValidationHarnessMilliseconds = validationTimer.ElapsedMilliSeconds
};
}
private static void ValidateReference(
TinyStoriesConfig config,
Gpt2ByteBpeTokenizer tokenizer,
LlmReferenceData historical,
GreedyGenerationReferenceDocument reference )
{
RequireExactArray( "reference prompt IDs", historical.InputTokenIds, reference.InputTokenIds );
RequireExactArray(
"historical generated IDs", historical.GeneratedTokenIds,
reference.GeneratedTokenIds );
if ( historical.Prompt != reference.Prompt ||
historical.GeneratedText != reference.GeneratedText )
{
throw new InvalidOperationException(
"[LLM:ERROR] Compact generation reference differs from historical reference.json." );
}
if ( reference.EosTokenId != config.EosTokenId ||
reference.BosTokenId != config.BosTokenId )
{
throw new InvalidOperationException(
$"[LLM:ERROR] Generation special IDs expected BOS/EOS " +
$"{config.BosTokenId}/{config.EosTokenId}, found " +
$"{reference.BosTokenId}/{reference.EosTokenId}." );
}
if ( reference.DoSample || reference.BosAutomaticallyAdded ||
reference.EosAutomaticallyAdded || !reference.RawLogitsUsedForArgmax ||
!reference.CachedAndUncachedSequencesMatch || reference.CsharpBaselineUseCache ||
reference.LogitsProcessors is null || reference.LogitsProcessors.Length != 0 ||
reference.ArgmaxTieBreak != "first (lowest) vocabulary index" )
{
throw new InvalidOperationException(
"[LLM:ERROR] Generation reference does not describe raw, uncached, " +
"deterministic first-index argmax semantics." );
}
if ( reference.MaxNewTokens <= 0 || reference.Steps is null ||
reference.Steps.Length != reference.GeneratedTokenIds.Length ||
reference.Steps.Length > reference.MaxNewTokens )
{
throw new InvalidOperationException(
"[LLM:ERROR] Generation reference step/token counts are inconsistent." );
}
if ( tokenizer.Decode( reference.GeneratedTokenIds ) != reference.GeneratedText )
{
throw new InvalidOperationException(
"[LLM:ERROR] C# tokenizer cannot reproduce Python generated text from " +
"the authoritative token sequence." );
}
if ( reference.LaterFullLogitReference is null ||
reference.LaterFullLogitReference.Step < 1 ||
reference.LaterFullLogitReference.Step >= reference.Steps.Length ||
reference.LaterFullLogitReference.Elements != config.VocabularySize ||
reference.LaterFullLogitReference.Bytes != config.VocabularySize * sizeof( float ) )
{
throw new InvalidOperationException(
"[LLM:ERROR] Later generation full-logit reference metadata is invalid." );
}
}
private static void ValidateStepContext(
GreedyGenerationReferenceStep expected,
int[] actualInput )
{
if ( expected.Step < 0 || expected.InputTokenIds is null ||
expected.InputSequenceLength != expected.InputTokenIds.Length ||
expected.NewTokenPosition != expected.InputSequenceLength )
{
throw new InvalidOperationException(
$"[LLM:ERROR] Generation reference step {expected.Step} context metadata is invalid." );
}
RequireExactArray( $"generation step {expected.Step} input", expected.InputTokenIds, actualInput );
}
private static void ValidateTopFive(
Gpt2ByteBpeTokenizer tokenizer,
GreedyGenerationReferenceStep expected,
GreedyGenerationStepResult actual )
{
if ( expected.TopFive is null || expected.TopFive.Length != 5 ||
actual.TopFive is null || actual.TopFive.Length != 5 )
{
throw new InvalidOperationException(
$"[LLM:ERROR] Generation step {actual.Step} requires five top-logit entries." );
}
for ( int rank = 0; rank < 5; rank++ )
{
ForwardReferenceTopLogit expectedRank = expected.TopFive[rank];
LogitRank actualRank = actual.TopFive[rank];
string decoded = tokenizer.Decode( new[] { actualRank.TokenId } );
if ( expectedRank.Rank != actualRank.Rank ||
expectedRank.TokenId != actualRank.TokenId ||
expectedRank.DecodedToken != decoded ||
Math.Abs( expectedRank.Logit - actualRank.Logit ) > TopLogitAbsoluteTolerance )
{
throw new InvalidOperationException(
$"[LLM:ERROR] Generation step {actual.Step} top-5 rank {rank + 1} " +
$"expected token={expectedRank.TokenId} logit={expectedRank.Logit:G9} " +
$"decoded='{EscapeVisible( expectedRank.DecodedToken )}', actual " +
$"token={actualRank.TokenId} logit={actualRank.Logit:G9} " +
$"decoded='{EscapeVisible( decoded )}'." );
}
}
}
private static void LogMismatch(
Gpt2ByteBpeTokenizer tokenizer,
GreedyGenerationReferenceStep expected,
GreedyGenerationStepObservation observation )
{
GreedyGenerationStepResult actual = observation.Step;
LlmLog.Error(
$"Greedy mismatch step={actual.Step} context={actual.InputSequenceLength} " +
$"input=[{string.Join( ",", observation.InputTokenIds )}] " +
$"expected={expected.ExpectedNextTokenId} " +
$"expected_piece='{EscapeVisible( expected.DecodedToken )}' actual={actual.TokenId} " +
$"actual_piece='{EscapeVisible( tokenizer.Decode( new[] { actual.TokenId } ) )}' " +
$"expected_margin={expected.Top1Top2Margin:G9} " +
$"actual_margin={actual.Top1Top2Margin:G9}." );
LlmLog.Error(
$"Python top5=[{string.Join( ",", expected.TopFive.Select( item => $"{item.TokenId}:{item.Logit:G9}" ) )}] " +
$"C# top5=[{string.Join( ",", actual.TopFive.Select( item => $"{item.TokenId}:{item.Logit:G9}" ) )}]. " +
"Generate a full Python logit reference for this step only before changing tolerances." );
}
private static void RequireExactArray( string label, int[] expected, int[] actual )
{
if ( expected is null || actual is null || expected.Length != actual.Length )
{
throw new InvalidOperationException(
$"[LLM:ERROR] {label} expected length {expected?.Length ?? -1}, " +
$"actual {actual?.Length ?? -1}." );
}
for ( int index = 0; index < expected.Length; index++ )
{
if ( expected[index] != actual[index] )
{
throw new InvalidOperationException(
$"[LLM:ERROR] {label} mismatch at index {index}: " +
$"expected={expected[index]}, actual={actual[index]}." );
}
}
}
private static string EscapeVisible( string value )
{
return (value ?? "<null>")
.Replace( "\\", "\\\\" )
.Replace( "\r", "\\r" )
.Replace( "\n", "\\n" )
.Replace( "\t", "\\t" )
.Replace( "'", "\\'" );
}
}
Game
game
namespace LlmPoc.Llm;
public static class LlmLog
{
public static bool TraceEnabled { get; set; }
public static void Info( string category, string message )
{
Log.Info( $"[LLM:{category}] {message}" );
}
public static void Trace( string category, string message )
{
if ( TraceEnabled )
{
Log.Info( $"[LLM:{category}] {message}" );
}
}
public static void Warning( string category, string message )
{
Log.Warning( $"[LLM:{category}] {message}" );
}
public static void Error( string message )
{
Log.Error( $"[LLM:ERROR] {message}" );
}
}
Game
game
using System.Text;
namespace LlmPoc.Llm;
public sealed class TensorSummary
{
public string Name { get; init; }
public string Shape { get; init; }
public long ElementCount { get; init; }
public float Minimum { get; init; }
public float Maximum { get; init; }
public double Mean { get; init; }
public double StandardDeviation { get; init; }
public double Rms { get; init; }
public int NaNCount { get; init; }
public int PositiveInfinityCount { get; init; }
public int NegativeInfinityCount { get; init; }
public string FirstValues { get; init; }
public string LastValues { get; init; }
public ulong Checksum { get; init; }
public bool IsFinite => NaNCount == 0 && PositiveInfinityCount == 0 && NegativeInfinityCount == 0;
public long FiniteCount => ElementCount - NaNCount - PositiveInfinityCount - NegativeInfinityCount;
public override string ToString()
{
return $"name={Name} shape={Shape} count={ElementCount:N0} " +
$"min={Minimum:G9} max={Maximum:G9} mean={Mean:G12} " +
$"std={StandardDeviation:G12} rms={Rms:G12} finite={FiniteCount:N0}/{ElementCount:N0} " +
$"nan={NaNCount} +inf={PositiveInfinityCount} -inf={NegativeInfinityCount} " +
$"first={FirstValues} last={LastValues} fnv1a64={Checksum:X16}";
}
}
public sealed class NumericComparison
{
public int ElementCount { get; init; }
public double MaximumAbsoluteError { get; init; }
public double MeanAbsoluteError { get; init; }
public double MaximumRelativeError { get; init; }
public int MaximumErrorIndex { get; init; }
public float ExpectedAtMaximumError { get; init; }
public float ActualAtMaximumError { get; init; }
public int FirstFailingIndex { get; init; }
public float ExpectedAtFirstFailure { get; init; }
public float ActualAtFirstFailure { get; init; }
public double AbsoluteTolerance { get; init; }
public double RelativeTolerance { get; init; }
public bool Passed { get; init; }
public override string ToString()
{
string failure = FirstFailingIndex < 0
? ""
: $" first_fail_index={FirstFailingIndex} " +
$"first_expected={ExpectedAtFirstFailure:G9} first_actual={ActualAtFirstFailure:G9}";
return $"count={ElementCount:N0} max_abs={MaximumAbsoluteError:G12} " +
$"mean_abs={MeanAbsoluteError:G12} max_rel={MaximumRelativeError:G12} " +
$"max_index={MaximumErrorIndex} expected={ExpectedAtMaximumError:G9} " +
$"actual={ActualAtMaximumError:G9} abs_tol={AbsoluteTolerance:G6} " +
$"rel_tol={RelativeTolerance:G6}{failure} {(Passed ? "PASS" : "FAIL")}";
}
}
public static class TensorDiagnostics
{
private const ulong FnvOffsetBasis = 14695981039346656037UL;
private const ulong FnvPrime = 1099511628211UL;
public static TensorSummary Summarize( Tensor tensor, int edgeCount = 4 )
{
if ( tensor is null )
{
throw new ArgumentNullException( nameof( tensor ) );
}
return Summarize( tensor.Name, tensor.ShapeText, tensor.Data, edgeCount );
}
public static TensorSummary Summarize(
string name,
string shape,
ReadOnlySpan<float> values,
int edgeCount = 4 )
{
if ( edgeCount < 0 || edgeCount > 32 )
{
throw new ArgumentOutOfRangeException(
nameof( edgeCount ), edgeCount, "[LLM:ERROR] Edge count must be between 0 and 32." );
}
if ( values.Length == 0 )
{
throw new ArgumentException( $"[LLM:ERROR] {name} cannot be summarized because it is empty." );
}
float minimum = float.PositiveInfinity;
float maximum = float.NegativeInfinity;
double sum = 0;
double sumSquares = 0;
int finiteCount = 0;
int nanCount = 0;
int positiveInfinityCount = 0;
int negativeInfinityCount = 0;
ulong checksum = FnvOffsetBasis;
for ( int index = 0; index < values.Length; index++ )
{
float value = values[index];
uint bits = unchecked( (uint)BitConverter.SingleToInt32Bits( value ) );
checksum = HashByte( checksum, (byte)bits );
checksum = HashByte( checksum, (byte)(bits >> 8) );
checksum = HashByte( checksum, (byte)(bits >> 16) );
checksum = HashByte( checksum, (byte)(bits >> 24) );
if ( float.IsNaN( value ) )
{
nanCount++;
continue;
}
if ( float.IsPositiveInfinity( value ) )
{
positiveInfinityCount++;
continue;
}
if ( float.IsNegativeInfinity( value ) )
{
negativeInfinityCount++;
continue;
}
minimum = Math.Min( minimum, value );
maximum = Math.Max( maximum, value );
sum += value;
sumSquares += (double)value * value;
finiteCount++;
}
double mean = finiteCount == 0 ? double.NaN : sum / finiteCount;
double rms = finiteCount == 0 ? double.NaN : Math.Sqrt( sumSquares / finiteCount );
double variance = finiteCount == 0 ? double.NaN : Math.Max( 0, sumSquares / finiteCount - mean * mean );
return new TensorSummary
{
Name = name,
Shape = shape,
ElementCount = values.Length,
Minimum = finiteCount == 0 ? float.NaN : minimum,
Maximum = finiteCount == 0 ? float.NaN : maximum,
Mean = mean,
StandardDeviation = Math.Sqrt( variance ),
Rms = rms,
NaNCount = nanCount,
PositiveInfinityCount = positiveInfinityCount,
NegativeInfinityCount = negativeInfinityCount,
FirstValues = FormatEdge( values, 0, Math.Min( edgeCount, values.Length ) ),
LastValues = FormatEdge( values, Math.Max( 0, values.Length - edgeCount ), Math.Min( edgeCount, values.Length ) ),
Checksum = checksum
};
}
public static NumericComparison Compare(
ReadOnlySpan<float> expected,
ReadOnlySpan<float> actual,
double absoluteTolerance = 1e-4,
double relativeTolerance = 1e-3 )
{
if ( expected.Length != actual.Length )
{
throw new ArgumentException(
$"[LLM:ERROR] Parity comparison shape mismatch: expected {expected.Length:N0} " +
$"elements, actual {actual.Length:N0}." );
}
if ( expected.Length == 0 )
{
throw new ArgumentException( "[LLM:ERROR] Parity comparison cannot use empty arrays." );
}
if ( absoluteTolerance < 0 || relativeTolerance < 0 )
{
throw new ArgumentOutOfRangeException(
nameof( absoluteTolerance ), "[LLM:ERROR] Parity tolerances cannot be negative." );
}
double maxAbsolute = -1;
double maxRelative = 0;
double absoluteSum = 0;
int maxIndex = 0;
int firstFailingIndex = -1;
bool passed = true;
for ( int index = 0; index < expected.Length; index++ )
{
float expectedValue = expected[index];
float actualValue = actual[index];
bool finite = float.IsFinite( expectedValue ) && float.IsFinite( actualValue );
if ( !finite )
{
passed = false;
if ( firstFailingIndex < 0 )
{
firstFailingIndex = index;
}
}
double absolute = Math.Abs( (double)actualValue - expectedValue );
double scale = Math.Max( Math.Abs( expectedValue ), 1e-12 );
double relative = absolute / scale;
absoluteSum += absolute;
maxRelative = Math.Max( maxRelative, relative );
if ( absolute > maxAbsolute )
{
maxAbsolute = absolute;
maxIndex = index;
}
if ( absolute > absoluteTolerance + relativeTolerance * Math.Abs( expectedValue ) )
{
passed = false;
if ( firstFailingIndex < 0 )
{
firstFailingIndex = index;
}
}
}
return new NumericComparison
{
ElementCount = expected.Length,
MaximumAbsoluteError = maxAbsolute,
MeanAbsoluteError = absoluteSum / expected.Length,
MaximumRelativeError = maxRelative,
MaximumErrorIndex = maxIndex,
ExpectedAtMaximumError = expected[maxIndex],
ActualAtMaximumError = actual[maxIndex],
FirstFailingIndex = firstFailingIndex,
ExpectedAtFirstFailure = firstFailingIndex < 0 ? 0 : expected[firstFailingIndex],
ActualAtFirstFailure = firstFailingIndex < 0 ? 0 : actual[firstFailingIndex],
AbsoluteTolerance = absoluteTolerance,
RelativeTolerance = relativeTolerance,
Passed = passed
};
}
private static ulong HashByte( ulong hash, byte value )
{
return (hash ^ value) * FnvPrime;
}
private static string FormatEdge( ReadOnlySpan<float> values, int start, int count )
{
StringBuilder builder = new();
builder.Append( '[' );
for ( int index = 0; index < count; index++ )
{
if ( index > 0 )
{
builder.Append( ',' );
}
builder.Append( values[start + index].ToString( "G9" ) );
}
builder.Append( ']' );
return builder.ToString();
}
}
Game
game
namespace LlmPoc.Llm;
public static class TinyStoriesForwardStages
{
public const string TokenEmbeddingName = "transformer.wte.weight";
public const string PositionEmbeddingName = "transformer.wpe.weight";
public const string Layer0Ln1WeightName = "transformer.h.0.ln_1.weight";
public const string Layer0Ln1BiasName = "transformer.h.0.ln_1.bias";
public const string Layer0QWeightName = "transformer.h.0.attn.attention.q_proj.weight";
public const string Layer0KWeightName = "transformer.h.0.attn.attention.k_proj.weight";
public const string Layer0VWeightName = "transformer.h.0.attn.attention.v_proj.weight";
public const string Layer0AttentionOutProjectionWeightName =
"transformer.h.0.attn.attention.out_proj.weight";
public const string Layer0AttentionOutProjectionBiasName =
"transformer.h.0.attn.attention.out_proj.bias";
public const string Layer0Ln2WeightName = "transformer.h.0.ln_2.weight";
public const string Layer0Ln2BiasName = "transformer.h.0.ln_2.bias";
public const string Layer0MlpFcWeightName = "transformer.h.0.mlp.c_fc.weight";
public const string Layer0MlpFcBiasName = "transformer.h.0.mlp.c_fc.bias";
public const string Layer0MlpProjWeightName = "transformer.h.0.mlp.c_proj.weight";
public const string Layer0MlpProjBiasName = "transformer.h.0.mlp.c_proj.bias";
public const float GeluNewTanhCoefficient = 0.7978845608028654f;
public const float GeluNewCubicCoefficient = 0.044715f;
public const string GeluNewFormula =
"0.5*x*(1.0+tanh(sqrt(2.0/pi)*(x+0.044715*pow(x,3.0))))";
public static Tensor CombineEmbeddings(
SboxLlmModel model,
TinyStoriesConfig config,
IReadOnlyList<int> tokenIds,
bool logDiagnostics = true )
{
if ( model is null )
{
throw new ArgumentNullException( nameof( model ) );
}
if ( config is null )
{
throw new ArgumentNullException( nameof( config ) );
}
if ( tokenIds is null || tokenIds.Count == 0 )
{
throw new ArgumentException( "[LLM:ERROR] Embedding input token IDs cannot be empty." );
}
if ( tokenIds.Count > config.MaximumPositions )
{
throw new InvalidOperationException(
$"[LLM:ERROR] Embedding sequence length {tokenIds.Count} exceeds " +
$"maximum positions {config.MaximumPositions}." );
}
Tensor tokenEmbedding = model.GetRequiredTensor( TokenEmbeddingName );
Tensor positionEmbedding = model.GetRequiredTensor( PositionEmbeddingName );
tokenEmbedding.RequireShape( config.VocabularySize, config.HiddenSize );
positionEmbedding.RequireShape( config.MaximumPositions, config.HiddenSize );
if ( tokenIds.Count > int.MaxValue / config.HiddenSize )
{
throw new InvalidOperationException(
$"[LLM:ERROR] Embedding output shape [{tokenIds.Count},{config.HiddenSize}] " +
"exceeds the managed array limit." );
}
if ( logDiagnostics )
{
LlmLog.Info(
"EMBED",
$"input shape=[{tokenIds.Count}] tokens=[{string.Join( ",", tokenIds )}] " +
$"positions=[0..{tokenIds.Count - 1}] token_tensor={TokenEmbeddingName}{tokenEmbedding.ShapeText} " +
$"position_tensor={PositionEmbeddingName}{positionEmbedding.ShapeText}" );
}
int hiddenSize = config.HiddenSize;
float[] output = new float[tokenIds.Count * hiddenSize];
for ( int position = 0; position < tokenIds.Count; position++ )
{
int tokenId = tokenIds[position];
if ( tokenId < 0 || tokenId >= config.VocabularySize )
{
throw new IndexOutOfRangeException(
$"[LLM:ERROR] Embedding token ID {tokenId} at sequence index {position} " +
$"is outside [0,{config.VocabularySize})." );
}
int tokenRow = tokenId * hiddenSize;
int positionRow = position * hiddenSize;
int outputRow = position * hiddenSize;
for ( int hidden = 0; hidden < hiddenSize; hidden++ )
{
output[outputRow + hidden] =
tokenEmbedding.Data[tokenRow + hidden] +
positionEmbedding.Data[positionRow + hidden];
}
}
return new Tensor(
"forward.combined_embedding",
new[] { tokenIds.Count, hiddenSize },
output );
}
public static Tensor ApplyLayer0Ln1(
SboxLlmModel model,
TinyStoriesConfig config,
Tensor input )
{
return ApplyLayerNorm(
model,
config,
input,
Layer0Ln1WeightName,
Layer0Ln1BiasName,
"layer=0 ln=1",
"Layer 0 ln_1",
"forward.layer0.ln_1" );
}
public static Tensor ApplyLayer0Ln2(
SboxLlmModel model,
TinyStoriesConfig config,
Tensor input )
{
return ApplyLayerNorm(
model,
config,
input,
Layer0Ln2WeightName,
Layer0Ln2BiasName,
"layer=0 ln=2",
"Layer 0 ln_2",
"forward.layer0.ln_2" );
}
public static Tensor ApplyLayerNorm(
SboxLlmModel model,
TinyStoriesConfig config,
Tensor input,
int layerIndex,
int normIndex,
string outputName,
bool logDiagnostics = true )
{
if ( layerIndex < 0 || layerIndex >= config.LayerCount )
{
throw new IndexOutOfRangeException(
$"[LLM:ERROR] LayerNorm layer index {layerIndex} is outside " +
$"[0,{config.LayerCount})." );
}
if ( normIndex != 1 && normIndex != 2 )
{
throw new ArgumentOutOfRangeException(
nameof( normIndex ), normIndex,
"[LLM:ERROR] GPT-Neo transformer LayerNorm index must be 1 or 2." );
}
string prefix = $"transformer.h.{layerIndex}.ln_{normIndex}";
return ApplyLayerNorm(
model,
config,
input,
$"{prefix}.weight",
$"{prefix}.bias",
$"layer={layerIndex} ln={normIndex}",
$"Layer {layerIndex} ln_{normIndex}",
outputName,
logDiagnostics );
}
public static Tensor ApplyFinalLayerNorm(
SboxLlmModel model,
TinyStoriesConfig config,
Tensor input,
string outputName,
bool logDiagnostics = true )
{
return ApplyLayerNorm(
model,
config,
input,
"transformer.ln_f.weight",
"transformer.ln_f.bias",
"stage=final_ln",
"Final model LayerNorm",
outputName,
logDiagnostics );
}
private static Tensor ApplyLayerNorm(
SboxLlmModel model,
TinyStoriesConfig config,
Tensor input,
string weightName,
string biasName,
string logLabel,
string errorLabel,
string outputName,
bool logDiagnostics = true )
{
if ( model is null ) throw new ArgumentNullException( nameof( model ) );
if ( config is null ) throw new ArgumentNullException( nameof( config ) );
if ( input is null ) throw new ArgumentNullException( nameof( input ) );
if ( input.Rank != 2 || input.Shape[0] <= 0 || input.Shape[1] != config.HiddenSize )
{
throw new InvalidOperationException(
$"[LLM:ERROR] {errorLabel} input expected shape [sequence,{config.HiddenSize}] " +
$"with a positive sequence length, found {input.ShapeText}." );
}
Tensor weight = model.GetRequiredTensor( weightName );
Tensor bias = model.GetRequiredTensor( biasName );
weight.RequireShape( config.HiddenSize );
bias.RequireShape( config.HiddenSize );
if ( !(config.LayerNormEpsilon > 0) || !float.IsFinite( config.LayerNormEpsilon ) )
{
throw new InvalidOperationException(
$"[LLM:ERROR] {errorLabel} epsilon must be positive and finite, " +
$"found {config.LayerNormEpsilon:G9}." );
}
int sequenceLength = input.Shape[0];
int hiddenSize = config.HiddenSize;
float[] output = new float[input.Data.Length];
if ( logDiagnostics )
{
LlmLog.Info(
"LN",
$"{logLabel} input_shape={input.ShapeText} expected_shape=[{sequenceLength},{hiddenSize}] " +
$"weight={weightName}{weight.ShapeText} bias={biasName}{bias.ShapeText} " +
$"epsilon={config.LayerNormEpsilon:G9} variance=population(unbiased=false)" );
}
for ( int token = 0; token < sequenceLength; token++ )
{
int row = token * hiddenSize;
// PyTorch 2.9.1's installed AVX2 LayerNorm kernel uses RowwiseMoments:
// eight FP32 Welford lanes over this model's 64 hidden values, followed
// by a left-to-right cascade of the lane moments. This scalar spelling
// reproduces that numerical reduction order without introducing SIMD.
(float mean, float variance) = ComputeAvx2RowwiseMoments64( input.Data, row );
float denominator = MathF.Sqrt( variance + config.LayerNormEpsilon );
if ( !(denominator > 0) || !float.IsFinite( denominator ) )
{
throw new InvalidOperationException(
$"[LLM:ERROR] {errorLabel} token {token} produced invalid denominator " +
$"{denominator:G9} from mean={mean:G9}, variance={variance:G9}, " +
$"epsilon={config.LayerNormEpsilon:G9}." );
}
float inverseStandardDeviation = 1.0f / denominator;
for ( int hidden = 0; hidden < hiddenSize; hidden++ )
{
float normalized = (input.Data[row + hidden] - mean) * inverseStandardDeviation;
output[row + hidden] = normalized * weight.Data[hidden] + bias.Data[hidden];
}
if ( logDiagnostics && token == 0 )
{
LlmLog.Trace(
"LN",
$"{logLabel} token=0 mean={mean:G12} variance={variance:G12} " +
$"denominator={denominator:G12} inverse_std={inverseStandardDeviation:G12} " +
$"output_first=[{output[row]:G9},{output[row + 1]:G9}," +
$"{output[row + 2]:G9},{output[row + 3]:G9}]" );
}
}
return new Tensor(
outputName,
new[] { sequenceLength, hiddenSize },
output );
}
private static (float Mean, float Variance) ComputeAvx2RowwiseMoments64(
float[] values,
int rowOffset )
{
const int laneCount = 8;
const int valuesPerLane = 8;
float[] laneMeans = new float[laneCount];
float[] laneMoment2 = new float[laneCount];
for ( int item = 0; item < valuesPerLane; item++ )
{
float reciprocalCount = 1.0f / (item + 1);
int itemOffset = rowOffset + item * laneCount;
for ( int lane = 0; lane < laneCount; lane++ )
{
float value = values[itemOffset + lane];
float delta = value - laneMeans[lane];
float meanIncrement = delta * reciprocalCount;
laneMeans[lane] += meanIncrement;
float remainingDelta = value - laneMeans[lane];
float momentIncrement = delta * remainingDelta;
laneMoment2[lane] += momentIncrement;
}
}
int accumulatedCount = 0;
float mean = 0.0f;
float moment2 = 0.0f;
for ( int lane = 0; lane < laneCount; lane++ )
{
int combinedCount = accumulatedCount + valuesPerLane;
float contribution = (float)valuesPerLane / combinedCount;
float delta = laneMeans[lane] - mean;
float meanIncrement = contribution * delta;
mean += meanIncrement;
float deltaSquared = delta * delta;
float weightedDelta = deltaSquared * contribution;
weightedDelta *= accumulatedCount;
float combinedMoment = laneMoment2[lane] + weightedDelta;
moment2 += combinedMoment;
accumulatedCount = combinedCount;
}
return (mean, moment2 / 64.0f);
}
public static Tensor LinearNoBias( Tensor input, Tensor weight, string outputName )
{
return Linear( input, weight, null, outputName );
}
public static Tensor LinearWithBias(
Tensor input,
Tensor weight,
Tensor bias,
string outputName )
{
if ( bias is null )
{
throw new ArgumentNullException( nameof( bias ) );
}
return Linear( input, weight, bias, outputName );
}
private static Tensor Linear(
Tensor input,
Tensor weight,
Tensor bias,
string outputName )
{
if ( input is null )
{
throw new ArgumentNullException( nameof( input ) );
}
if ( weight is null )
{
throw new ArgumentNullException( nameof( weight ) );
}
if ( string.IsNullOrWhiteSpace( outputName ) )
{
throw new ArgumentException(
"[LLM:ERROR] Linear projection output name cannot be empty.",
nameof( outputName ) );
}
if ( input.Rank != 2 )
{
throw new InvalidOperationException(
$"[LLM:ERROR] {outputName} linear input expected rank 2 " +
$"[sequence,input_size], found {input.ShapeText}." );
}
if ( weight.Rank != 2 )
{
throw new InvalidOperationException(
$"[LLM:ERROR] {outputName} weight '{weight.Name}' expected rank 2 " +
$"[output_size,input_size], found {weight.ShapeText}." );
}
int sequenceLength = input.Shape[0];
int inputSize = input.Shape[1];
int outputSize = weight.Shape[0];
if ( weight.Shape[1] != inputSize )
{
throw new InvalidOperationException(
$"[LLM:ERROR] {outputName} incompatible linear shapes: input={input.ShapeText}, " +
$"weight={weight.ShapeText}; weight input axis expected {inputSize}, " +
$"found {weight.Shape[1]}." );
}
if ( bias is not null )
{
if ( bias.Rank != 1 || bias.Shape[0] != outputSize )
{
throw new InvalidOperationException(
$"[LLM:ERROR] {outputName} linear bias '{bias.Name}' expected shape " +
$"[{outputSize}], found {bias.ShapeText}." );
}
}
if ( sequenceLength > int.MaxValue / outputSize )
{
throw new InvalidOperationException(
$"[LLM:ERROR] {outputName} output shape [{sequenceLength},{outputSize}] " +
"exceeds the managed array limit." );
}
float[] output = new float[sequenceLength * outputSize];
for ( int token = 0; token < sequenceLength; token++ )
{
int inputRow = token * inputSize;
int outputRow = token * outputSize;
for ( int outputFeature = 0; outputFeature < outputSize; outputFeature++ )
{
int weightRow = outputFeature * inputSize;
// The installed PyTorch 2.9.1 CPU Linear path was independently
// checked against the live module outputs. For this model it is
// bit-identical to a left-to-right FP32 fused multiply-add reduction.
// The small attention matmuls intentionally retain their separately
// validated non-fused multiply/add loops.
float sum = 0.0f;
for ( int inputFeature = 0; inputFeature < inputSize; inputFeature++ )
{
sum = MathF.FusedMultiplyAdd(
input.Data[inputRow + inputFeature],
weight.Data[weightRow + inputFeature],
sum );
}
output[outputRow + outputFeature] =
bias is null ? sum : sum + bias.Data[outputFeature];
}
}
return new Tensor(
outputName,
new[] { sequenceLength, outputSize },
output );
}
public static string DescribeLinearDotProduct(
Tensor input,
Tensor weight,
int token,
int outputFeature,
float expected,
float actual,
Tensor bias = null )
{
if ( input is null || weight is null || input.Rank != 2 || weight.Rank != 2 ||
weight.Shape[1] != input.Shape[1] )
{
throw new InvalidOperationException(
"[LLM:ERROR] Dot-product diagnostic requires compatible rank-2 input and weight tensors." );
}
if ( bias is not null && (bias.Rank != 1 || bias.Shape[0] != weight.Shape[0]) )
{
throw new InvalidOperationException(
$"[LLM:ERROR] Dot-product diagnostic bias '{bias.Name}' expected shape " +
$"[{weight.Shape[0]}], found {bias.ShapeText}." );
}
if ( token < 0 || token >= input.Shape[0] ||
outputFeature < 0 || outputFeature >= weight.Shape[0] )
{
throw new IndexOutOfRangeException(
$"[LLM:ERROR] Dot-product diagnostic index token={token}, " +
$"output_feature={outputFeature} is outside input={input.ShapeText}, " +
$"weight={weight.ShapeText}." );
}
int inputSize = input.Shape[1];
int inputRow = token * inputSize;
int weightRow = outputFeature * inputSize;
float sum = 0;
float product0 = input.Data[inputRow] * weight.Data[weightRow];
float product1 = input.Data[inputRow + 1] * weight.Data[weightRow + 1];
float product2 = input.Data[inputRow + 2] * weight.Data[weightRow + 2];
float product3 = input.Data[inputRow + 3] * weight.Data[weightRow + 3];
for ( int inputFeature = 0; inputFeature < inputSize; inputFeature++ )
{
sum = MathF.FusedMultiplyAdd(
input.Data[inputRow + inputFeature],
weight.Data[weightRow + inputFeature],
sum );
}
float biasValue = bias is null ? 0 : bias.Data[outputFeature];
float recomputed = bias is null ? sum : sum + biasValue;
string formula = bias is null
? "sum(input[token,i]*weight[out,i])"
: "sum(input[token,i]*weight[out,i])+bias[out]";
return $"token={token} output_feature={outputFeature} input_length={inputSize} " +
$"formula={formula} expected={expected:G9} actual={actual:G9} " +
$"dot={sum:G9} bias={biasValue:G9} recomputed={recomputed:G9} " +
$"first_products=[{product0:G9},{product1:G9},{product2:G9},{product3:G9}]";
}
public static Tensor ApplyGeluNew( Tensor input, string outputName )
{
if ( input is null )
{
throw new ArgumentNullException( nameof( input ) );
}
if ( string.IsNullOrWhiteSpace( outputName ) )
{
throw new ArgumentException(
"[LLM:ERROR] GELU-new output name cannot be empty.",
nameof( outputName ) );
}
if ( input.Rank != 2 || input.Shape[0] <= 0 || input.Shape[1] <= 0 )
{
throw new InvalidOperationException(
$"[LLM:ERROR] {outputName} GELU-new input expected a positive rank-2 " +
$"[sequence,feature] tensor, found {input.ShapeText}." );
}
float[] output = new float[input.Data.Length];
for ( int index = 0; index < input.Data.Length; index++ )
{
float value = input.Data[index];
if ( !float.IsFinite( value ) )
{
throw new InvalidOperationException(
$"[LLM:ERROR] {outputName} GELU-new input contains non-finite " +
$"value at flat index {index}: {value}." );
}
float cube = MathF.Pow( value, 3.0f );
float inner = value + GeluNewCubicCoefficient * cube;
float tanhArgument = GeluNewTanhCoefficient * inner;
float tanhValue = MathF.Tanh( tanhArgument );
float activated = (0.5f * value) * (1.0f + tanhValue);
if ( !float.IsFinite( activated ) )
{
throw new InvalidOperationException(
$"[LLM:ERROR] {outputName} GELU-new produced non-finite output " +
$"at flat index {index}: input={value:G9}, cube={cube:G9}, " +
$"tanh_argument={tanhArgument:G9}, output={activated}." );
}
output[index] = activated;
}
return new Tensor(
outputName,
new[] { input.Shape[0], input.Shape[1] },
output );
}
public static string DescribeGeluNewElement(
Tensor input,
Tensor output,
int token,
int feature,
float expected )
{
if ( input is null || output is null || input.Rank != 2 || output.Rank != 2 ||
input.Shape[0] != output.Shape[0] || input.Shape[1] != output.Shape[1] )
{
throw new InvalidOperationException(
"[LLM:ERROR] GELU-new diagnostic requires matching rank-2 tensors." );
}
if ( token < 0 || token >= input.Shape[0] || feature < 0 || feature >= input.Shape[1] )
{
throw new IndexOutOfRangeException(
$"[LLM:ERROR] GELU-new diagnostic index [{token},{feature}] is " +
$"outside {input.ShapeText}." );
}
int index = token * input.Shape[1] + feature;
float value = input.Data[index];
float cube = MathF.Pow( value, 3.0f );
float inner = value + GeluNewCubicCoefficient * cube;
float tanhArgument = GeluNewTanhCoefficient * inner;
float tanhValue = MathF.Tanh( tanhArgument );
float recomputed = (0.5f * value) * (1.0f + tanhValue);
float actual = output.Data[index];
return $"token={token} feature={feature} input={value:G9} cube={cube:G9} " +
$"cubic_coefficient={GeluNewCubicCoefficient:G9} inner={inner:G9} " +
$"tanh_coefficient={GeluNewTanhCoefficient:G9} " +
$"tanh_argument={tanhArgument:G9} tanh={tanhValue:G9} " +
$"recomputed={recomputed:G9} actual={actual:G9} python={expected:G9} " +
$"abs_diff={MathF.Abs( actual - expected ):G12}";
}
public static Tensor AddMlpResidual(
Tensor residualSource,
Tensor mlpBranch,
string outputName )
{
if ( residualSource is null )
{
throw new ArgumentNullException( nameof( residualSource ) );
}
if ( mlpBranch is null )
{
throw new ArgumentNullException( nameof( mlpBranch ) );
}
if ( string.IsNullOrWhiteSpace( outputName ) )
{
throw new ArgumentException(
"[LLM:ERROR] MLP residual output name cannot be empty.",
nameof( outputName ) );
}
if ( residualSource.Rank != 2 || mlpBranch.Rank != 2 ||
residualSource.Shape[0] != mlpBranch.Shape[0] ||
residualSource.Shape[1] != mlpBranch.Shape[1] ||
residualSource.Data.Length != mlpBranch.Data.Length )
{
throw new InvalidOperationException(
$"[LLM:ERROR] {outputName} MLP residual addition requires matching " +
$"rank-2 tensors, found residual={residualSource.ShapeText}, " +
$"MLP={mlpBranch.ShapeText}." );
}
float[] output = new float[residualSource.Data.Length];
for ( int index = 0; index < output.Length; index++ )
{
// Match GPTNeoBlock.forward: residual + feed_forward_hidden_states.
output[index] = residualSource.Data[index] + mlpBranch.Data[index];
}
return new Tensor(
outputName,
new[] { residualSource.Shape[0], residualSource.Shape[1] },
output );
}
public static string DescribeMlpResidualAddition(
Tensor residualSource,
Tensor mlpBranch,
Tensor output,
int token,
int feature,
float expected )
{
if ( residualSource is null || mlpBranch is null || output is null ||
residualSource.Rank != 2 || mlpBranch.Rank != 2 || output.Rank != 2 ||
residualSource.Shape[0] != mlpBranch.Shape[0] ||
residualSource.Shape[1] != mlpBranch.Shape[1] ||
residualSource.Shape[0] != output.Shape[0] ||
residualSource.Shape[1] != output.Shape[1] )
{
throw new InvalidOperationException(
"[LLM:ERROR] MLP residual diagnostic requires matching rank-2 tensors." );
}
if ( token < 0 || token >= output.Shape[0] || feature < 0 || feature >= output.Shape[1] )
{
throw new IndexOutOfRangeException(
$"[LLM:ERROR] MLP residual diagnostic index [{token},{feature}] is " +
$"outside {output.ShapeText}." );
}
int index = token * output.Shape[1] + feature;
float residual = residualSource.Data[index];
float branch = mlpBranch.Data[index];
float actual = output.Data[index];
return $"token={token} feature={feature} residual_source={residual:G9} " +
$"mlp_branch={branch:G9} sum={actual:G9} python={expected:G9} " +
$"abs_diff={MathF.Abs( expected - actual ):G12}";
}
public static Tensor AddResidual(
Tensor attentionBranch,
Tensor residualSource,
string outputName )
{
if ( attentionBranch is null )
{
throw new ArgumentNullException( nameof( attentionBranch ) );
}
if ( residualSource is null )
{
throw new ArgumentNullException( nameof( residualSource ) );
}
if ( string.IsNullOrWhiteSpace( outputName ) )
{
throw new ArgumentException(
"[LLM:ERROR] Residual-add output name cannot be empty.",
nameof( outputName ) );
}
if ( attentionBranch.Rank != 2 || residualSource.Rank != 2 ||
attentionBranch.Shape[0] != residualSource.Shape[0] ||
attentionBranch.Shape[1] != residualSource.Shape[1] ||
attentionBranch.Data.Length != residualSource.Data.Length )
{
throw new InvalidOperationException(
$"[LLM:ERROR] {outputName} residual addition requires matching rank-2 " +
$"tensors, found attention={attentionBranch.ShapeText}, " +
$"residual={residualSource.ShapeText}." );
}
float[] output = new float[attentionBranch.Data.Length];
for ( int index = 0; index < output.Length; index++ )
{
output[index] = attentionBranch.Data[index] + residualSource.Data[index];
}
return new Tensor(
outputName,
new[] { attentionBranch.Shape[0], attentionBranch.Shape[1] },
output );
}
public static string DescribeResidualAddition(
Tensor attentionBranch,
Tensor residualSource,
Tensor output,
int token,
int feature,
float expected )
{
if ( attentionBranch is null || residualSource is null || output is null ||
attentionBranch.Rank != 2 || residualSource.Rank != 2 || output.Rank != 2 ||
attentionBranch.Shape[0] != residualSource.Shape[0] ||
attentionBranch.Shape[1] != residualSource.Shape[1] ||
attentionBranch.Shape[0] != output.Shape[0] ||
attentionBranch.Shape[1] != output.Shape[1] )
{
throw new InvalidOperationException(
"[LLM:ERROR] Residual diagnostic requires compatible rank-2 attention, " +
"residual, and output tensors." );
}
if ( token < 0 || token >= output.Shape[0] ||
feature < 0 || feature >= output.Shape[1] )
{
throw new IndexOutOfRangeException(
$"[LLM:ERROR] Residual diagnostic index [{token},{feature}] is " +
$"outside output shape {output.ShapeText}." );
}
int index = token * output.Shape[1] + feature;
float branch = attentionBranch.Data[index];
float residual = residualSource.Data[index];
float actual = output.Data[index];
return $"token={token} feature={feature} residual_source={residual:G9} " +
$"attention_branch={branch:G9} sum={actual:G9} python={expected:G9} " +
$"abs_diff={MathF.Abs( expected - actual ):G12}";
}
public static Tensor SplitHeads( Tensor projection, int headCount, string outputName )
{
if ( projection is null )
{
throw new ArgumentNullException( nameof( projection ) );
}
if ( string.IsNullOrWhiteSpace( outputName ) )
{
throw new ArgumentException(
"[LLM:ERROR] Head-split output name cannot be empty.",
nameof( outputName ) );
}
if ( projection.Rank != 2 || projection.Shape[0] <= 0 || projection.Shape[1] <= 0 )
{
throw new InvalidOperationException(
$"[LLM:ERROR] {outputName} head split expected projection shape " +
$"[sequence,hidden] with positive dimensions, found {projection.ShapeText}." );
}
if ( headCount <= 0 || projection.Shape[1] % headCount != 0 )
{
throw new InvalidOperationException(
$"[LLM:ERROR] {outputName} hidden size {projection.Shape[1]} must be divisible " +
$"by positive head count {headCount}." );
}
int sequenceLength = projection.Shape[0];
int hiddenSize = projection.Shape[1];
int headDimension = hiddenSize / headCount;
float[] output = new float[projection.Data.Length];
for ( int head = 0; head < headCount; head++ )
{
for ( int token = 0; token < sequenceLength; token++ )
{
int sourceRow = token * hiddenSize;
int destinationRow = (head * sequenceLength + token) * headDimension;
for ( int component = 0; component < headDimension; component++ )
{
int sourceFeature = head * headDimension + component;
output[destinationRow + component] = projection.Data[sourceRow + sourceFeature];
}
}
}
return new Tensor(
outputName,
new[] { headCount, sequenceLength, headDimension },
output );
}
public static string DescribeHeadMapping(
Tensor projection,
Tensor heads,
int head,
int token,
int component )
{
if ( projection is null || heads is null || projection.Rank != 2 || heads.Rank != 3 )
{
throw new InvalidOperationException(
"[LLM:ERROR] Head mapping diagnostic requires rank-2 projection and rank-3 heads." );
}
int headCount = heads.Shape[0];
int sequenceLength = heads.Shape[1];
int headDimension = heads.Shape[2];
if ( projection.Shape[0] != sequenceLength ||
projection.Shape[1] != headCount * headDimension )
{
throw new InvalidOperationException(
$"[LLM:ERROR] Head mapping diagnostic incompatible shapes: " +
$"projection={projection.ShapeText}, heads={heads.ShapeText}." );
}
if ( head < 0 || head >= headCount || token < 0 || token >= sequenceLength ||
component < 0 || component >= headDimension )
{
throw new IndexOutOfRangeException(
$"[LLM:ERROR] Head mapping diagnostic index [{head},{token},{component}] " +
$"is outside {heads.ShapeText}." );
}
int sourceFeature = head * headDimension + component;
int sourceIndex = token * projection.Shape[1] + sourceFeature;
int destinationIndex = (head * sequenceLength + token) * headDimension + component;
float source = projection.Data[sourceIndex];
float destination = heads.Data[destinationIndex];
return $"head={head} token={token} component={component} source_feature={sourceFeature} " +
$"source_index={sourceIndex} destination_index={destinationIndex} " +
$"source={source:G9} destination={destination:G9} " +
$"source_bits={BitConverter.SingleToInt32Bits( source ):X8} " +
$"destination_bits={BitConverter.SingleToInt32Bits( destination ):X8}";
}
public static Tensor MergeHeads( Tensor contextHeads, string outputName )
{
if ( contextHeads is null )
{
throw new ArgumentNullException( nameof( contextHeads ) );
}
if ( string.IsNullOrWhiteSpace( outputName ) )
{
throw new ArgumentException(
"[LLM:ERROR] Head-merge output name cannot be empty.",
nameof( outputName ) );
}
if ( contextHeads.Rank != 3 || contextHeads.Shape[0] <= 0 ||
contextHeads.Shape[1] <= 0 || contextHeads.Shape[2] <= 0 )
{
throw new InvalidOperationException(
$"[LLM:ERROR] {outputName} head merge expected positive shape " +
$"[head,sequence,head_dimension], found {contextHeads.ShapeText}." );
}
int headCount = contextHeads.Shape[0];
int sequenceLength = contextHeads.Shape[1];
int headDimension = contextHeads.Shape[2];
if ( headCount > int.MaxValue / headDimension )
{
throw new InvalidOperationException(
$"[LLM:ERROR] {outputName} hidden size {headCount}*{headDimension} " +
"exceeds the managed array limit." );
}
int hiddenSize = headCount * headDimension;
if ( sequenceLength > int.MaxValue / hiddenSize )
{
throw new InvalidOperationException(
$"[LLM:ERROR] {outputName} output shape [{sequenceLength},{hiddenSize}] " +
"exceeds the managed array limit." );
}
float[] output = new float[sequenceLength * hiddenSize];
for ( int token = 0; token < sequenceLength; token++ )
{
int destinationRow = token * hiddenSize;
for ( int head = 0; head < headCount; head++ )
{
int sourceRow = (head * sequenceLength + token) * headDimension;
int destinationHead = destinationRow + head * headDimension;
for ( int component = 0; component < headDimension; component++ )
{
output[destinationHead + component] =
contextHeads.Data[sourceRow + component];
}
}
}
return new Tensor(
outputName,
new[] { sequenceLength, hiddenSize },
output );
}
public static string DescribeHeadMergeMapping(
Tensor contextHeads,
Tensor merged,
int token,
int feature )
{
if ( contextHeads is null || merged is null ||
contextHeads.Rank != 3 || merged.Rank != 2 )
{
throw new InvalidOperationException(
"[LLM:ERROR] Head-merge diagnostic requires rank-3 context heads and " +
"rank-2 merged context." );
}
int headCount = contextHeads.Shape[0];
int sequenceLength = contextHeads.Shape[1];
int headDimension = contextHeads.Shape[2];
int hiddenSize = headCount * headDimension;
if ( merged.Shape[0] != sequenceLength || merged.Shape[1] != hiddenSize )
{
throw new InvalidOperationException(
$"[LLM:ERROR] Head-merge diagnostic incompatible shapes: " +
$"context={contextHeads.ShapeText}, merged={merged.ShapeText}." );
}
if ( token < 0 || token >= sequenceLength || feature < 0 || feature >= hiddenSize )
{
throw new IndexOutOfRangeException(
$"[LLM:ERROR] Head-merge diagnostic index token={token}, feature={feature} " +
$"is outside merged shape {merged.ShapeText}." );
}
int head = feature / headDimension;
int component = feature % headDimension;
int sourceIndex = (head * sequenceLength + token) * headDimension + component;
int destinationIndex = token * hiddenSize + feature;
float source = contextHeads.Data[sourceIndex];
float destination = merged.Data[destinationIndex];
return $"token={token} feature={feature} head={head} component={component} " +
$"source_index={sourceIndex} destination_index={destinationIndex} " +
$"source={source:G9} merged={destination:G9} " +
$"source_bits={BitConverter.SingleToInt32Bits( source ):X8} " +
$"merged_bits={BitConverter.SingleToInt32Bits( destination ):X8}";
}
public static Tensor ComputeScaledUnmaskedAttentionScores(
Tensor queryHeads,
Tensor keyHeads,
float scale,
string outputName )
{
if ( queryHeads is null )
{
throw new ArgumentNullException( nameof( queryHeads ) );
}
if ( keyHeads is null )
{
throw new ArgumentNullException( nameof( keyHeads ) );
}
if ( string.IsNullOrWhiteSpace( outputName ) )
{
throw new ArgumentException(
"[LLM:ERROR] Attention-score output name cannot be empty.",
nameof( outputName ) );
}
if ( queryHeads.Rank != 3 || keyHeads.Rank != 3 )
{
throw new InvalidOperationException(
$"[LLM:ERROR] {outputName} expected Q/K heads rank 3 " +
$"[head,sequence,component], found Q={queryHeads.ShapeText}, " +
$"K={keyHeads.ShapeText}." );
}
if ( queryHeads.Shape[0] != keyHeads.Shape[0] ||
queryHeads.Shape[2] != keyHeads.Shape[2] )
{
throw new InvalidOperationException(
$"[LLM:ERROR] {outputName} Q/K head count and component dimensions must match; " +
$"found Q={queryHeads.ShapeText}, K={keyHeads.ShapeText}." );
}
if ( !float.IsFinite( scale ) || !(scale > 0) )
{
throw new InvalidOperationException(
$"[LLM:ERROR] {outputName} attention scale must be positive and finite, " +
$"found {scale:G9}." );
}
int headCount = queryHeads.Shape[0];
int queryLength = queryHeads.Shape[1];
int keyLength = keyHeads.Shape[1];
int headDimension = queryHeads.Shape[2];
if ( headCount > int.MaxValue / queryLength ||
headCount * queryLength > int.MaxValue / keyLength )
{
throw new InvalidOperationException(
$"[LLM:ERROR] {outputName} shape [{headCount},{queryLength},{keyLength}] " +
"exceeds the managed array limit." );
}
float[] output = new float[headCount * queryLength * keyLength];
for ( int head = 0; head < headCount; head++ )
{
for ( int query = 0; query < queryLength; query++ )
{
int queryRow = (head * queryLength + query) * headDimension;
for ( int key = 0; key < keyLength; key++ )
{
int keyRow = (head * keyLength + key) * headDimension;
float rawSum = 0;
for ( int component = 0; component < headDimension; component++ )
{
rawSum += queryHeads.Data[queryRow + component] *
keyHeads.Data[keyRow + component];
}
// Transformers 5.15 GPT-Neo performs no inverse-sqrt scaling.
// Avoid adding an operation when the exact model-equivalent factor is 1.
output[(head * queryLength + query) * keyLength + key] =
scale == 1.0f ? rawSum : rawSum * scale;
}
}
}
return new Tensor(
outputName,
new[] { headCount, queryLength, keyLength },
output );
}
public static bool IsLayer0AttentionAllowed(
string attentionType,
int query,
int key,
int queryLength,
int keyLength )
{
return IsAttentionAllowed(
attentionType,
windowSize: 0,
query,
key,
queryLength,
keyLength );
}
public static bool IsAttentionAllowed(
string attentionType,
int windowSize,
int query,
int key,
int queryLength,
int keyLength )
{
if ( attentionType != "global" && attentionType != "local" )
{
throw new InvalidOperationException(
$"[LLM:ERROR] GPT-Neo attention type must be global or local, " +
$"found '{attentionType}'." );
}
if ( attentionType == "local" && windowSize <= 0 )
{
throw new InvalidOperationException(
$"[LLM:ERROR] Local GPT-Neo attention window must be positive, " +
$"found {windowSize}." );
}
if ( queryLength <= 0 || keyLength <= 0 || queryLength > keyLength ||
query < 0 || query >= queryLength || key < 0 || key >= keyLength )
{
throw new IndexOutOfRangeException(
$"[LLM:ERROR] Attention mask index query={query}, key={key} is invalid " +
$"for query_length={queryLength}, key_length={keyLength}. " +
"The verified source slice requires 0 < query_length <= key_length." );
}
int absoluteQuery = keyLength - queryLength + query;
bool causal = key <= absoluteQuery;
return attentionType == "global"
? causal
: causal && absoluteQuery - key < windowSize;
}
public static Tensor ApplyLayer0AttentionMask(
Tensor unmaskedScores,
string attentionType,
float maskedSentinel,
string outputName )
{
return ApplyAttentionMask(
unmaskedScores,
attentionType,
windowSize: 0,
maskedSentinel,
outputName );
}
public static Tensor ApplyAttentionMask(
Tensor unmaskedScores,
string attentionType,
int windowSize,
float maskedSentinel,
string outputName )
{
if ( unmaskedScores is null )
{
throw new ArgumentNullException( nameof( unmaskedScores ) );
}
if ( string.IsNullOrWhiteSpace( outputName ) )
{
throw new ArgumentException(
"[LLM:ERROR] Masked attention-score output name cannot be empty.",
nameof( outputName ) );
}
if ( unmaskedScores.Rank != 3 || unmaskedScores.Shape[0] <= 0 ||
unmaskedScores.Shape[1] <= 0 || unmaskedScores.Shape[2] <= 0 ||
unmaskedScores.Shape[1] > unmaskedScores.Shape[2] )
{
throw new InvalidOperationException(
$"[LLM:ERROR] {outputName} expected unmasked scores shape " +
$"[head,query,key] with 0 < query <= key, found " +
$"{unmaskedScores.ShapeText}." );
}
if ( !float.IsFinite( maskedSentinel ) || !(maskedSentinel < 0) )
{
throw new InvalidOperationException(
$"[LLM:ERROR] {outputName} mask sentinel must be finite and negative, " +
$"found {maskedSentinel:G9}." );
}
int headCount = unmaskedScores.Shape[0];
int queryLength = unmaskedScores.Shape[1];
int keyLength = unmaskedScores.Shape[2];
float[] output = new float[unmaskedScores.Data.Length];
for ( int head = 0; head < headCount; head++ )
{
for ( int query = 0; query < queryLength; query++ )
{
for ( int key = 0; key < keyLength; key++ )
{
int index = (head * queryLength + query) * keyLength + key;
bool allowed = IsAttentionAllowed(
attentionType,
windowSize,
query,
key,
queryLength,
keyLength );
output[index] = allowed ? unmaskedScores.Data[index] : maskedSentinel;
}
}
}
return new Tensor(
outputName,
new[] { headCount, queryLength, keyLength },
output );
}
public static Tensor ComputeAttentionProbabilities(
Tensor maskedScores,
string outputName )
{
if ( maskedScores is null )
{
throw new ArgumentNullException( nameof( maskedScores ) );
}
if ( string.IsNullOrWhiteSpace( outputName ) )
{
throw new ArgumentException(
"[LLM:ERROR] Attention-probability output name cannot be empty.",
nameof( outputName ) );
}
if ( maskedScores.Rank != 3 || maskedScores.Shape[0] <= 0 ||
maskedScores.Shape[1] <= 0 || maskedScores.Shape[2] <= 0 )
{
throw new InvalidOperationException(
$"[LLM:ERROR] {outputName} expected masked scores shape " +
$"[head,query,key] with positive dimensions, found " +
$"{maskedScores.ShapeText}." );
}
int headCount = maskedScores.Shape[0];
int queryLength = maskedScores.Shape[1];
int keyLength = maskedScores.Shape[2];
float[] output = new float[maskedScores.Data.Length];
for ( int head = 0; head < headCount; head++ )
{
for ( int query = 0; query < queryLength; query++ )
{
int row = (head * queryLength + query) * keyLength;
float rowMaximum = float.NegativeInfinity;
for ( int key = 0; key < keyLength; key++ )
{
float value = maskedScores.Data[row + key];
if ( !float.IsFinite( value ) )
{
throw new InvalidOperationException(
$"[LLM:ERROR] {outputName} input contains a non-finite score " +
$"at [{head},{query},{key}]: {value}." );
}
rowMaximum = MathF.Max( rowMaximum, value );
}
float exponentialSum = 0;
for ( int key = 0; key < keyLength; key++ )
{
float shifted = maskedScores.Data[row + key] - rowMaximum;
float exponential = MathF.Exp( shifted );
if ( !float.IsFinite( exponential ) || exponential < 0 )
{
throw new InvalidOperationException(
$"[LLM:ERROR] {outputName} produced invalid exp at " +
$"[{head},{query},{key}]: input={maskedScores.Data[row + key]:G9}, " +
$"maximum={rowMaximum:G9}, shifted={shifted:G9}, " +
$"exp={exponential:G9}." );
}
output[row + key] = exponential;
exponentialSum += exponential;
}
if ( !(exponentialSum > 0) || !float.IsFinite( exponentialSum ) )
{
throw new InvalidOperationException(
$"[LLM:ERROR] {outputName} row [{head},{query}] produced invalid " +
$"exponential sum {exponentialSum:G9}." );
}
float inverseSum = 1.0f / exponentialSum;
for ( int key = 0; key < keyLength; key++ )
{
output[row + key] *= inverseSum;
}
}
}
return new Tensor(
outputName,
new[] { headCount, queryLength, keyLength },
output );
}
public static Tensor ComputeAttentionContextHeads(
Tensor probabilities,
Tensor valueHeads,
string outputName )
{
if ( probabilities is null )
{
throw new ArgumentNullException( nameof( probabilities ) );
}
if ( valueHeads is null )
{
throw new ArgumentNullException( nameof( valueHeads ) );
}
if ( string.IsNullOrWhiteSpace( outputName ) )
{
throw new ArgumentException(
"[LLM:ERROR] Attention-context output name cannot be empty.",
nameof( outputName ) );
}
if ( probabilities.Rank != 3 || probabilities.Shape[0] <= 0 ||
probabilities.Shape[1] <= 0 || probabilities.Shape[2] <= 0 )
{
throw new InvalidOperationException(
$"[LLM:ERROR] {outputName} expected probabilities shape " +
$"[head,query,key] with positive dimensions, found " +
$"{probabilities.ShapeText}." );
}
if ( valueHeads.Rank != 3 || valueHeads.Shape[0] <= 0 ||
valueHeads.Shape[1] <= 0 || valueHeads.Shape[2] <= 0 )
{
throw new InvalidOperationException(
$"[LLM:ERROR] {outputName} expected value shape " +
$"[head,key,component] with positive dimensions, found " +
$"{valueHeads.ShapeText}." );
}
if ( probabilities.Shape[0] != valueHeads.Shape[0] ||
probabilities.Shape[2] != valueHeads.Shape[1] )
{
throw new InvalidOperationException(
$"[LLM:ERROR] {outputName} incompatible probabilities/value shapes: " +
$"probabilities={probabilities.ShapeText} [head,query,key], " +
$"V={valueHeads.ShapeText} [head,key,component]." );
}
int headCount = probabilities.Shape[0];
int queryLength = probabilities.Shape[1];
int keyLength = probabilities.Shape[2];
int headDimension = valueHeads.Shape[2];
float[] output = new float[headCount * queryLength * headDimension];
for ( int head = 0; head < headCount; head++ )
{
for ( int query = 0; query < queryLength; query++ )
{
int probabilityRow = (head * queryLength + query) * keyLength;
for ( int component = 0; component < headDimension; component++ )
{
float sum = 0;
for ( int key = 0; key < keyLength; key++ )
{
float probability = probabilities.Data[probabilityRow + key];
int valueIndex = (head * keyLength + key) * headDimension + component;
float value = valueHeads.Data[valueIndex];
if ( !float.IsFinite( probability ) || !float.IsFinite( value ) )
{
throw new InvalidOperationException(
$"[LLM:ERROR] {outputName} received non-finite input at " +
$"head={head}, query={query}, key={key}, component={component}: " +
$"probability={probability:G9}, V={value:G9}." );
}
sum += probability * value;
}
if ( !float.IsFinite( sum ) )
{
throw new InvalidOperationException(
$"[LLM:ERROR] {outputName} produced non-finite context at " +
$"[{head},{query},{component}]: {sum:G9}." );
}
int outputIndex =
(head * queryLength + query) * headDimension + component;
output[outputIndex] = sum;
}
}
}
return new Tensor(
outputName,
new[] { headCount, queryLength, headDimension },
output );
}
public static string DescribeAttentionContextElement(
Tensor probabilities,
Tensor valueHeads,
Tensor context,
int head,
int query,
int component,
float expected )
{
if ( probabilities is null || valueHeads is null || context is null ||
probabilities.Rank != 3 || valueHeads.Rank != 3 || context.Rank != 3 ||
probabilities.Shape[0] != valueHeads.Shape[0] ||
probabilities.Shape[0] != context.Shape[0] ||
probabilities.Shape[1] != context.Shape[1] ||
probabilities.Shape[2] != valueHeads.Shape[1] ||
valueHeads.Shape[2] != context.Shape[2] )
{
throw new InvalidOperationException(
"[LLM:ERROR] Attention-context diagnostic requires compatible " +
"probability [head,query,key], V [head,key,component], and context " +
"[head,query,component] tensors." );
}
if ( head < 0 || head >= context.Shape[0] ||
query < 0 || query >= context.Shape[1] ||
component < 0 || component >= context.Shape[2] )
{
throw new IndexOutOfRangeException(
$"[LLM:ERROR] Attention-context diagnostic index " +
$"[{head},{query},{component}] is outside {context.ShapeText}." );
}
int keyLength = probabilities.Shape[2];
int headDimension = valueHeads.Shape[2];
int probabilityRow = (head * probabilities.Shape[1] + query) * keyLength;
float[] probabilityValues = new float[keyLength];
float[] values = new float[keyLength];
float[] products = new float[keyLength];
float sum = 0;
for ( int key = 0; key < keyLength; key++ )
{
probabilityValues[key] = probabilities.Data[probabilityRow + key];
int valueIndex = (head * keyLength + key) * headDimension + component;
values[key] = valueHeads.Data[valueIndex];
products[key] = probabilityValues[key] * values[key];
sum += products[key];
}
int contextIndex =
(head * context.Shape[1] + query) * context.Shape[2] + component;
float actual = context.Data[contextIndex];
if ( BitConverter.SingleToInt32Bits( sum ) !=
BitConverter.SingleToInt32Bits( actual ) )
{
throw new InvalidOperationException(
$"[LLM:ERROR] Attention-context diagnostic recomputation differs at " +
$"[{head},{query},{component}]: recomputed={sum:G9}, actual={actual:G9}." );
}
return $"head={head} query={query} component={component} " +
$"probabilities={FormatFloatList( probabilityValues )} " +
$"V={FormatFloatList( values )} products={FormatFloatList( products )} " +
$"accumulated={actual:G9} python={expected:G9} " +
$"abs_diff={MathF.Abs( actual - expected ):G12}";
}
public static string DescribeAttentionSoftmaxRow(
Tensor maskedScores,
Tensor probabilities,
Tensor expectedProbabilities,
int head,
int query )
{
if ( maskedScores is null || probabilities is null || expectedProbabilities is null ||
maskedScores.Rank != 3 || probabilities.Rank != 3 || expectedProbabilities.Rank != 3 ||
maskedScores.Shape[0] != probabilities.Shape[0] ||
maskedScores.Shape[1] != probabilities.Shape[1] ||
maskedScores.Shape[2] != probabilities.Shape[2] ||
maskedScores.Shape[0] != expectedProbabilities.Shape[0] ||
maskedScores.Shape[1] != expectedProbabilities.Shape[1] ||
maskedScores.Shape[2] != expectedProbabilities.Shape[2] )
{
throw new InvalidOperationException(
"[LLM:ERROR] Softmax row diagnostic requires matching rank-3 tensors." );
}
if ( head < 0 || head >= maskedScores.Shape[0] ||
query < 0 || query >= maskedScores.Shape[1] )
{
throw new IndexOutOfRangeException(
$"[LLM:ERROR] Softmax row diagnostic index [{head},{query}] is outside " +
$"{maskedScores.ShapeText}." );
}
int keyLength = maskedScores.Shape[2];
int row = (head * maskedScores.Shape[1] + query) * keyLength;
float rowMaximum = float.NegativeInfinity;
for ( int key = 0; key < keyLength; key++ )
{
rowMaximum = MathF.Max( rowMaximum, maskedScores.Data[row + key] );
}
float[] inputs = new float[keyLength];
float[] shifted = new float[keyLength];
float[] exponentials = new float[keyLength];
float[] actual = new float[keyLength];
float[] expected = new float[keyLength];
float exponentialSum = 0;
float probabilitySum = 0;
for ( int key = 0; key < keyLength; key++ )
{
inputs[key] = maskedScores.Data[row + key];
shifted[key] = inputs[key] - rowMaximum;
exponentials[key] = MathF.Exp( shifted[key] );
exponentialSum += exponentials[key];
actual[key] = probabilities.Data[row + key];
expected[key] = expectedProbabilities.Data[row + key];
probabilitySum += actual[key];
}
return $"head={head} query={query} input={FormatFloatList( inputs )} " +
$"max={rowMaximum:G9} shifted={FormatFloatList( shifted )} " +
$"exp={FormatFloatList( exponentials )} exp_sum={exponentialSum:G9} " +
$"actual={FormatFloatList( actual )} python={FormatFloatList( expected )} " +
$"row_sum={probabilitySum:G9}";
}
public static string DescribeAttentionMaskApplication(
Tensor unmaskedScores,
Tensor maskedScores,
string attentionType,
float maskedSentinel,
int head,
int query,
int key )
{
if ( unmaskedScores is null || maskedScores is null ||
unmaskedScores.Rank != 3 || maskedScores.Rank != 3 ||
unmaskedScores.Shape[0] != maskedScores.Shape[0] ||
unmaskedScores.Shape[1] != maskedScores.Shape[1] ||
unmaskedScores.Shape[2] != maskedScores.Shape[2] )
{
throw new InvalidOperationException(
"[LLM:ERROR] Mask diagnostic requires matching rank-3 unmasked/masked tensors." );
}
int queryLength = unmaskedScores.Shape[1];
int keyLength = unmaskedScores.Shape[2];
if ( head < 0 || head >= unmaskedScores.Shape[0] )
{
throw new IndexOutOfRangeException(
$"[LLM:ERROR] Mask diagnostic head {head} is outside " +
$"{unmaskedScores.ShapeText}." );
}
bool allowed = IsLayer0AttentionAllowed(
attentionType,
query,
key,
queryLength,
keyLength );
int index = (head * queryLength + query) * keyLength + key;
float unmasked = unmaskedScores.Data[index];
float masked = maskedScores.Data[index];
return $"head={head} query={query} key={key} allowed={allowed} " +
$"unmasked={unmasked:G9} masked={masked:G9} " +
$"masked_bits=0x{BitConverter.SingleToInt32Bits( masked ):X8} " +
$"expected_sentinel={maskedSentinel:G9} " +
$"sentinel_bits=0x{BitConverter.SingleToInt32Bits( maskedSentinel ):X8}";
}
public static string DescribeAttentionScore(
Tensor queryHeads,
Tensor keyHeads,
Tensor scores,
int head,
int query,
int key,
float scale,
float expected )
{
if ( queryHeads is null || keyHeads is null || scores is null ||
queryHeads.Rank != 3 || keyHeads.Rank != 3 || scores.Rank != 3 )
{
throw new InvalidOperationException(
"[LLM:ERROR] Attention-score diagnostic requires rank-3 Q, K, and score tensors." );
}
int headDimension = queryHeads.Shape[2];
if ( headDimension != 4 || keyHeads.Shape[2] != headDimension ||
queryHeads.Shape[0] != keyHeads.Shape[0] ||
scores.Shape[0] != queryHeads.Shape[0] ||
scores.Shape[1] != queryHeads.Shape[1] ||
scores.Shape[2] != keyHeads.Shape[1] )
{
throw new InvalidOperationException(
$"[LLM:ERROR] Attention-score diagnostic expected compatible Q/K/scores with " +
$"head dimension 4, found Q={queryHeads.ShapeText}, K={keyHeads.ShapeText}, " +
$"scores={scores.ShapeText}." );
}
if ( head < 0 || head >= scores.Shape[0] || query < 0 || query >= scores.Shape[1] ||
key < 0 || key >= scores.Shape[2] )
{
throw new IndexOutOfRangeException(
$"[LLM:ERROR] Attention-score diagnostic index [{head},{query},{key}] " +
$"is outside {scores.ShapeText}." );
}
int queryRow = (head * queryHeads.Shape[1] + query) * headDimension;
int keyRow = (head * keyHeads.Shape[1] + key) * headDimension;
float q0 = queryHeads.Data[queryRow];
float q1 = queryHeads.Data[queryRow + 1];
float q2 = queryHeads.Data[queryRow + 2];
float q3 = queryHeads.Data[queryRow + 3];
float k0 = keyHeads.Data[keyRow];
float k1 = keyHeads.Data[keyRow + 1];
float k2 = keyHeads.Data[keyRow + 2];
float k3 = keyHeads.Data[keyRow + 3];
float p0 = q0 * k0;
float p1 = q1 * k1;
float p2 = q2 * k2;
float p3 = q3 * k3;
float rawSum = ((p0 + p1) + p2) + p3;
float recomputed = scale == 1.0f ? rawSum : rawSum * scale;
int scoreIndex = (head * scores.Shape[1] + query) * scores.Shape[2] + key;
float actual = scores.Data[scoreIndex];
if ( BitConverter.SingleToInt32Bits( recomputed ) !=
BitConverter.SingleToInt32Bits( actual ) )
{
throw new InvalidOperationException(
$"[LLM:ERROR] Attention-score diagnostic recomputation differs at " +
$"[{head},{query},{key}]: recomputed={recomputed:G9}, actual={actual:G9}." );
}
return $"head={head} query={query} key={key} " +
$"q=[{q0:G9},{q1:G9},{q2:G9},{q3:G9}] " +
$"k=[{k0:G9},{k1:G9},{k2:G9},{k3:G9}] " +
$"products=[{p0:G9},{p1:G9},{p2:G9},{p3:G9}] " +
$"raw_sum={rawSum:G9} scale={scale:G9} final={actual:G9} expected={expected:G9}";
}
private static string FormatFloatList( IReadOnlyList<float> values )
{
string[] formatted = new string[values.Count];
for ( int index = 0; index < values.Count; index++ )
{
formatted[index] = values[index].ToString( "G9" );
}
return $"[{string.Join( ",", formatted )}]";
}
}
Game
game
using Sandbox.Diagnostics;
namespace LlmPoc.Llm;
public sealed class GreedyGenerationStepResult
{
public int Step { get; init; }
public int InputSequenceLength { get; init; }
public int NewTokenPosition { get; init; }
public int TokenId { get; init; }
public string DecodedToken { get; init; }
public float Top1Logit { get; init; }
public int Top2TokenId { get; init; }
public float Top2Logit { get; init; }
public float Top1Top2Margin { get; init; }
public LogitRank[] TopFive { get; init; }
public bool EosReached { get; init; }
public double ForwardMilliseconds { get; init; }
public double LmHeadMilliseconds { get; init; }
public double StepMilliseconds { get; init; }
public double[] LayerMilliseconds { get; init; }
}
public sealed class GreedyGenerationStepObservation
{
public GreedyGenerationStepResult Step { get; init; }
public int[] InputTokenIds { get; init; }
public TinyStoriesModelForwardResult Forward { get; init; }
}
public sealed class GreedyGenerationResult
{
public int[] PromptTokenIds { get; init; }
public int[] GeneratedTokenIds { get; init; }
public int[] FullSequenceTokenIds { get; init; }
public string GeneratedText { get; init; }
public string StopReason { get; init; }
public bool EosReached { get; init; }
public GreedyGenerationStepResult[] Steps { get; init; }
public double TotalForwardMilliseconds { get; init; }
public double TotalGenerationMilliseconds { get; init; }
}
/// <summary>
/// Deterministic correctness baseline: every new token recomputes the complete
/// model over the complete current context. No K/V state is retained.
/// </summary>
public static class TinyStoriesGreedyGenerator
{
public static GreedyGenerationResult Generate(
SboxLlmModel model,
TinyStoriesConfig config,
Gpt2ByteBpeTokenizer tokenizer,
IReadOnlyList<int> promptTokenIds,
int maxNewTokens,
Action<GreedyGenerationStepObservation> observer = null,
bool logLifecycle = true )
{
if ( model is null ) throw new ArgumentNullException( nameof( model ) );
if ( config is null ) throw new ArgumentNullException( nameof( config ) );
if ( tokenizer is null ) throw new ArgumentNullException( nameof( tokenizer ) );
if ( promptTokenIds is null ) throw new ArgumentNullException( nameof( promptTokenIds ) );
if ( promptTokenIds.Count == 0 )
{
throw new ArgumentException(
"[LLM:ERROR] Greedy generation requires at least one prompt token.",
nameof( promptTokenIds ) );
}
if ( maxNewTokens <= 0 )
{
throw new ArgumentOutOfRangeException(
nameof( maxNewTokens ), maxNewTokens,
"[LLM:ERROR] Greedy generation maxNewTokens must be positive." );
}
if ( promptTokenIds.Count > config.MaximumPositions - maxNewTokens )
{
throw new InvalidOperationException(
$"[LLM:ERROR] Greedy generation prompt length {promptTokenIds.Count} plus " +
$"maxNewTokens {maxNewTokens} would exceed maximum context length " +
$"{config.MaximumPositions}; context truncation is disabled." );
}
int[] promptCopy = promptTokenIds.ToArray();
List<int> context = new( promptCopy.Length + maxNewTokens );
context.AddRange( promptCopy );
List<int> generated = new( maxNewTokens );
List<GreedyGenerationStepResult> steps = new( maxNewTokens );
double totalForwardMilliseconds = 0;
double totalGenerationMilliseconds = 0;
bool eosReached = false;
if ( logLifecycle )
{
LlmLog.Info(
"GEN",
$"starting prompt_tokens={promptCopy.Length} max_new_tokens={maxNewTokens} " +
$"maximum_context={config.MaximumPositions} strategy=greedy " +
"full_recompute=true kv_cache=false" );
}
for ( int stepIndex = 0; stepIndex < maxNewTokens; stepIndex++ )
{
FastTimer stepTimer = FastTimer.StartNew();
TinyStoriesModelForwardResult forward =
TinyStoriesModelForward.ForwardLastTokenLogits( model, config, context );
int tokenId = TinyStoriesModelHead.ArgmaxFinite(
forward.Logits.Data, $"generation.step{stepIndex}.logits" );
LogitRank[] topFive = TinyStoriesModelHead.TopKFinite(
forward.Logits.Data, 5, $"generation.step{stepIndex}.logits" );
string decodedToken = tokenizer.Decode( new[] { tokenId } );
float margin = topFive[0].Logit - topFive[1].Logit;
bool stepEos = tokenId == config.EosTokenId;
GreedyGenerationStepResult step = new()
{
Step = stepIndex,
InputSequenceLength = context.Count,
NewTokenPosition = context.Count,
TokenId = tokenId,
DecodedToken = decodedToken,
Top1Logit = topFive[0].Logit,
Top2TokenId = topFive[1].TokenId,
Top2Logit = topFive[1].Logit,
Top1Top2Margin = margin,
TopFive = topFive,
EosReached = stepEos,
ForwardMilliseconds = forward.TotalMilliseconds,
LmHeadMilliseconds = forward.LmHeadMilliseconds,
StepMilliseconds = stepTimer.ElapsedMilliSeconds,
LayerMilliseconds = forward.LayerMilliseconds
};
GreedyGenerationStepObservation observation = new()
{
Step = step,
InputTokenIds = context.ToArray(),
Forward = forward
};
observer?.Invoke( observation );
if ( observer is null && logLifecycle )
{
LlmLog.Info(
"GEN",
$"step={stepIndex} context={context.Count} token={tokenId} " +
$"piece='{EscapeVisible( decodedToken )}' margin={margin:G9} " +
$"forward_ms={forward.TotalMilliseconds:N4}" );
}
LlmLog.Trace(
"GEN",
$"step={stepIndex} sequence={context.Count} new_token_position={context.Count} " +
$"top5=[{string.Join( ",", topFive.Select( item => $"{item.TokenId}:{item.Logit:G9}" ) )}]" );
context.Add( tokenId );
generated.Add( tokenId );
steps.Add( step );
totalForwardMilliseconds += forward.TotalMilliseconds;
totalGenerationMilliseconds += step.StepMilliseconds;
if ( stepEos )
{
eosReached = true;
break;
}
}
for ( int index = 0; index < promptCopy.Length; index++ )
{
if ( promptCopy[index] != promptTokenIds[index] )
{
throw new InvalidOperationException(
$"[LLM:ERROR] Greedy generation mutated caller prompt token {index}." );
}
}
return new GreedyGenerationResult
{
PromptTokenIds = promptCopy,
GeneratedTokenIds = generated.ToArray(),
FullSequenceTokenIds = context.ToArray(),
GeneratedText = tokenizer.Decode( generated ),
StopReason = eosReached ? "eos" : "max_new_tokens",
EosReached = eosReached,
Steps = steps.ToArray(),
TotalForwardMilliseconds = totalForwardMilliseconds,
TotalGenerationMilliseconds = totalGenerationMilliseconds
};
}
private static string EscapeVisible( string value )
{
return value
.Replace( "\\", "\\\\" )
.Replace( "\r", "\\r" )
.Replace( "\n", "\\n" )
.Replace( "\t", "\\t" )
.Replace( "'", "\\'" );
}
}
Game
game
.button
{
position: relative;
> .button-right-column
{
flex-direction: column;
}
}
// default menu position is below
.button-hover-menu
{
position: absolute;
top: 100%;
flex-direction: column;
&.hidden
{
opacity: 0;
pointer-events: none;
}
}
Game
game
.dropdown
{
gap: 2px;
flex-grow: 1;
cursor: pointer;
justify-content: flex-end;
align-items: center;
padding: 0px 12px;
.button-right-column
{
flex-grow: 1;
}
}
Game
game
.package-list
{
flex-shrink: 1;
flex-wrap: wrap;
flex-grow: 1;
h1
{
width: 100%;
margin-top: 50px;
font-size: 40px;
}
PackageCard
{
&:hover
{
sound-in: "ui.button.over";
}
}
VirtualGrid
{
width: 100%;
height: 100%;
.cell
{
}
}
}
Game
game
namespace LlmPoc.Llm;
public static class ReferenceFloatData
{
public static float[] LoadFromMounted( string path, int expectedCount )
{
if ( !FileSystem.Mounted.FileExists( path ) )
{
throw new InvalidOperationException(
$"[LLM:ERROR] Reference FP32 file '{path}' is missing from FileSystem.Mounted." );
}
byte[] bytes = FileSystem.Mounted.ReadAllBytes( path ).ToArray();
if ( bytes.Length % sizeof( float ) != 0 )
{
throw new InvalidOperationException(
$"[LLM:ERROR] Reference FP32 file '{path}' has {bytes.Length:N0} bytes, " +
"which is not divisible by four." );
}
int count = bytes.Length / sizeof( float );
if ( count != expectedCount )
{
throw new InvalidOperationException(
$"[LLM:ERROR] Reference FP32 file '{path}' expected {expectedCount:N0} values " +
$"({expectedCount * sizeof( float ):N0} bytes), found {count:N0} values " +
$"({bytes.Length:N0} bytes)." );
}
float[] values = new float[count];
for ( int index = 0; index < count; index++ )
{
int offset = index * sizeof( float );
uint bits = (uint)(
bytes[offset]
| (bytes[offset + 1] << 8)
| (bytes[offset + 2] << 16)
| (bytes[offset + 3] << 24) );
values[index] = BitConverter.Int32BitsToSingle( unchecked( (int)bits ) );
}
return values;
}
public static int ArgmaxFinite( ReadOnlySpan<float> values, string logicalName )
{
if ( values.Length == 0 )
{
throw new ArgumentException(
$"[LLM:ERROR] {logicalName} cannot be argmaxed because it is empty." );
}
int bestIndex = 0;
float bestValue = values[0];
if ( !float.IsFinite( bestValue ) )
{
throw new InvalidOperationException(
$"[LLM:ERROR] {logicalName} contains non-finite value {bestValue} at index 0." );
}
for ( int index = 1; index < values.Length; index++ )
{
float value = values[index];
if ( !float.IsFinite( value ) )
{
throw new InvalidOperationException(
$"[LLM:ERROR] {logicalName} contains non-finite value {value} at index {index}." );
}
if ( value > bestValue )
{
bestValue = value;
bestIndex = index;
}
}
return bestIndex;
}
}
Game
game
using Sandbox.Diagnostics;
namespace LlmPoc.Llm;
public sealed class TinyStoriesModelForwardResult
{
public int SequenceLength { get; init; }
public Tensor FinalLayerNorm { get; init; }
public Tensor Logits { get; init; }
public double EmbeddingMilliseconds { get; init; }
public double[] LayerMilliseconds { get; init; }
public string[] AttentionTypes { get; init; }
public double AllLayersMilliseconds { get; init; }
public double FinalLayerNormMilliseconds { get; init; }
public double LmHeadMilliseconds { get; init; }
public double TotalMilliseconds { get; init; }
}
/// <summary>
/// Production-oriented full-context forward orchestration. It deliberately reuses
/// the parity-proven embedding, transformer-layer, final-norm, and LM-head kernels.
/// </summary>
public static class TinyStoriesModelForward
{
public static TinyStoriesModelForwardResult ForwardLastTokenLogits(
SboxLlmModel model,
TinyStoriesConfig config,
IReadOnlyList<int> tokenIds,
bool logDiagnostics = false )
{
if ( model is null ) throw new ArgumentNullException( nameof( model ) );
if ( config is null ) throw new ArgumentNullException( nameof( config ) );
if ( tokenIds is null ) throw new ArgumentNullException( nameof( tokenIds ) );
if ( tokenIds.Count == 0 )
{
throw new ArgumentException(
"[LLM:ERROR] Model forward requires at least one token.", nameof( tokenIds ) );
}
if ( tokenIds.Count > config.MaximumPositions )
{
throw new InvalidOperationException(
$"[LLM:ERROR] Model forward sequence length {tokenIds.Count} exceeds " +
$"maximum context length {config.MaximumPositions}; context truncation is disabled." );
}
for ( int index = 0; index < tokenIds.Count; index++ )
{
int tokenId = tokenIds[index];
if ( tokenId < 0 || tokenId >= config.VocabularySize )
{
throw new IndexOutOfRangeException(
$"[LLM:ERROR] Model forward token ID {tokenId} at index {index} is outside " +
$"[0,{config.VocabularySize})." );
}
}
FastTimer totalTimer = FastTimer.StartNew();
FastTimer embeddingTimer = FastTimer.StartNew();
Tensor hiddenStates = TinyStoriesForwardStages.CombineEmbeddings(
model, config, tokenIds, logDiagnostics );
double embeddingMilliseconds = embeddingTimer.ElapsedMilliSeconds;
double[] layerMilliseconds = new double[config.LayerCount];
string[] attentionTypes = new string[config.LayerCount];
double allLayersMilliseconds = 0;
for ( int layerIndex = 0; layerIndex < config.LayerCount; layerIndex++ )
{
GptNeoLayerForwardResult layer = GptNeoTransformerLayer.Forward(
model,
config,
hiddenStates,
layerIndex,
logDiagnostics,
validateInputMutation: false );
hiddenStates = layer.Output;
layerMilliseconds[layerIndex] = layer.ElapsedMilliseconds;
attentionTypes[layerIndex] = layer.AttentionType;
allLayersMilliseconds += layer.ElapsedMilliseconds;
}
FastTimer finalLayerNormTimer = FastTimer.StartNew();
Tensor finalLayerNorm = TinyStoriesForwardStages.ApplyFinalLayerNorm(
model, config, hiddenStates, "forward.final_ln", logDiagnostics );
double finalLayerNormMilliseconds = finalLayerNormTimer.ElapsedMilliSeconds;
FastTimer lmHeadTimer = FastTimer.StartNew();
Tensor logits = TinyStoriesModelHead.ProjectLastPositionNoBias(
finalLayerNorm,
model.GetRequiredTensor( "lm_head.weight" ),
"forward.final_logits_last_position" );
double lmHeadMilliseconds = lmHeadTimer.ElapsedMilliSeconds;
logits.RequireShape( config.VocabularySize );
if ( logDiagnostics )
{
LlmLog.Trace(
"PERF",
$"stage=model_forward sequence={tokenIds.Count} embeddings_ms={embeddingMilliseconds:N4} " +
$"layers_ms={allLayersMilliseconds:N4} final_ln_ms={finalLayerNormMilliseconds:N4} " +
$"lm_head_ms={lmHeadMilliseconds:N4} total_ms={totalTimer.ElapsedMilliSeconds:N4}" );
}
return new TinyStoriesModelForwardResult
{
SequenceLength = tokenIds.Count,
FinalLayerNorm = finalLayerNorm,
Logits = logits,
EmbeddingMilliseconds = embeddingMilliseconds,
LayerMilliseconds = layerMilliseconds,
AttentionTypes = attentionTypes,
AllLayersMilliseconds = allLayersMilliseconds,
FinalLayerNormMilliseconds = finalLayerNormMilliseconds,
LmHeadMilliseconds = lmHeadMilliseconds,
TotalMilliseconds = totalTimer.ElapsedMilliSeconds
};
}
}
Game
game
.switchcontrol
{
flex-direction: row;
width: 100px;
min-height: 24px;
align-items: center;
cursor: pointer;
.switch-frame
{
flex-grow: 0;
flex-shrink: 1;
width: 48px;
height: 16px;
background-color: #fff1;
margin: 0px 5px;
align-items: center;
border-radius: 100px;
transition: all 0.4s linear;
.switch-inner
{
position: relative;
flex-grow: 0;
flex-shrink: 1;
background-color: #999;
width: 25px;
height: 25px;
border-radius: 100px;
left: 20%;
transform: translateX( -50% );
transition: all 0.3s ease-out;
}
}
&.active
{
.switch-frame
{
background-color: #fffa;
}
.switch-inner
{
left: 80%;
background-color: #fff;
}
}
}
Game
game
$primary: red !default;
$primary-alt: white !default;
$switch-padding: 6px !default;
.button.popupbutton.dropdown
{
cursor: pointer;
transition: all .1s ease-out;
position: relative;
> .dropdown_indicator
{
position: absolute;
right: 8px;
}
&.open
{
border-bottom-left-radius: 1px;
border-bottom-right-radius: 1px;
transition: border-radius 0.2s ease-out;
}
}
select
{
min-height: 40px;
> option
{
display: none;
}
}
Game
game
menupanel
{
position: absolute;
z-index: 1000;
pointer-events: all;
font-size: 12px;
flex-shrink: 0;
.background
{
position: absolute;
left: -5000px;
right: -5000px;
top: -5000px;
bottom: -5000px;
}
> .inner
{
min-width: 200px;
min-height: 20px;
flex-direction: column;
font-family: Poppins;
font-weight: bold;
border-radius: 10px;
box-shadow: 5px 5px 30px #000e;
background-color: #2a2a2a;
flex-shrink: 0;
.spacer
{
height: 1px;
background-color: #0005;
}
.option
{
color: #fffa;
padding: 0px 8px;
cursor: pointer;
flex-shrink: 0;
height: 32px;
&:first-child
{
border-top-left-radius: 10px;
border-top-right-radius: 10px;
}
&:last-child
{
border-bottom-left-radius: 10px;
border-bottom-right-radius: 10px;
}
.icon
{
padding: 8px;
font-family: Material Icons;
justify-content: center;
align-items: center;
flex-shrink: 0;
}
.text
{
padding: 8px;
flex-shrink: 0;
}
&:hover
{
background-color: #3472e6;
color: #f5f8fe;
}
}
}
}
Debug: View Raw JSON Response
{
"TotalCount": 60,
"Files": [
{
"Ident": "jeffskitchen.llm_poc",
"Path": "ui/controls/vectorcontrol.cs.scss",
"FileName": "vectorcontrol.cs.scss",
"PackageType": "game",
"CodeKind": "Game",
"AssetVersionId": 342521,
"Code": "VectorControl\r\n{\r\n\tgap: 2px;\r\n\tflex-grow: 1;\r\n}\r\n\r\nVectorControl NumberEntry\r\n{\r\n\tflex-basis: 50%;\r\n}"
},
{
"Ident": "jeffskitchen.llm_poc",
"Path": "ui/controlsheet/controlsheetgroupheader.cs.scss",
"FileName": "controlsheetgroupheader.cs.scss",
"PackageType": "game",
"CodeKind": "Game",
"AssetVersionId": 342521,
"Code": "ControlSheetGroupHeader\r\n{\r\n\tfont-size: 1.33rem;\r\n\tcolor: red;\r\n\tgap: 2px;\r\n\talign-items: center;\r\n\r\n\t&.hidden\r\n\t{\r\n\t\tdisplay: none;\r\n\t}\r\n\r\n\t> .title\r\n\t{\r\n\t\tfont-weight: 600;\r\n\t}\r\n\r\n\t&.has-toggle\r\n\t{\r\n\t\tcursor: pointer;\r\n\t\topacity: 0.8;\r\n\r\n\t\t&:before\r\n\t\t{\r\n\t\t\tcontent: ' ';\r\n\t\t\twidth: 22px;\r\n\t\t\theight: 22px;\r\n\t\t\tbackground-color: #000a;\r\n\t\t\talign-items: center;\r\n\t\t\tjustify-content: center;\r\n\t\t\ttext-align: center;\r\n\t\t\tborder-radius: 5px;\r\n\t\t\tborder: 1px solid #555;\r\n\t\t}\r\n\r\n\t\t&:hover\r\n\t\t{\r\n\t\t\topacity: 1;\r\n\r\n\t\t\t&:before\r\n\t\t\t{\r\n\t\t\t\tborder-color: #888;\r\n\t\t\t}\r\n\t\t}\r\n\r\n\t\t&.checked\r\n\t\t{\r\n\t\t\t> .title\r\n\t\t\t{\r\n\t\t\t\tcolor: white;\r\n\t\t\t}\r\n\r\n\t\t\t&:before\r\n\t\t\t{\r\n\t\t\t\tcontent: '\u2713';\r\n\t\t\t\tfont-weight: bold;\r\n\t\t\t\tcolor: #08f;\r\n\t\t\t\tborder-color: #08f;\r\n\t\t\t}\r\n\t\t}\r\n\t}\r\n}\r\n"
},
{
"Ident": "jeffskitchen.llm_poc",
"Path": "styles/form/_checkbox.scss",
"FileName": "_checkbox.scss",
"PackageType": "game",
"CodeKind": "Game",
"AssetVersionId": 342521,
"Code": "\r\n@import \"/styles/_theme.scss\";\r\n\r\n$primary: $primary-blue !default;\r\n$primary-alt: white !default;\r\n$form-control-height: 24px !default;\r\n\r\n.checkbox\r\n{\r\n\tcursor: pointer;\r\n\tcolor: rgba( $primary-alt, 0.6 );\r\n\talign-items: center;\r\n\tgap: 8px;\r\n\r\n\tlabel\r\n\t{\r\n\t\tpointer-events: none;\r\n\t}\r\n\r\n\t> .checkmark\r\n\t{\r\n\t\tpadding: 1px;\r\n\t\tfont-size: 22px;\r\n\t\tborder: 1px solid $primary;\r\n\t\tborder-radius: $rounding-small;\r\n\t\ttext-align: center;\r\n\t\tjustify-content: center;\r\n\t\talign-items: center;\r\n\t\tcolor: transparent;\r\n\t\tmin-height: $form-control-height;\r\n\t\tpointer-events: none;\r\n\t\tflex-shrink: 0;\r\n\t}\r\n\r\n\t&.checked\r\n\t{\r\n\t\t> .checkmark\r\n\t\t{\r\n\t\t\tcolor: $primary-alt;\r\n\t\t\tbackground-color: $primary;\r\n\t\t}\r\n\t}\r\n\r\n\t&:active\r\n\t{\r\n\t\tcolor: $primary-alt;\r\n\t}\r\n\r\n\t&:hover\r\n\t{\r\n\t\tcolor: $primary-alt;\r\n\t}\r\n}\r\n"
},
{
"Ident": "jeffskitchen.llm_poc",
"Path": "Llm/GreedyGenerationReference.cs",
"FileName": "GreedyGenerationReference.cs",
"PackageType": "game",
"CodeKind": "Game",
"AssetVersionId": 342521,
"Code": "using System.Text.Json.Serialization;\n\nnamespace LlmPoc.Llm;\n\npublic sealed class GreedyGenerationReferenceDocument\n{\n\t[JsonPropertyName( \"model_id\" )]\n\tpublic string ModelId { get; set; }\n\n\t[JsonPropertyName( \"prompt\" )]\n\tpublic string Prompt { get; set; }\n\n\t[JsonPropertyName( \"input_token_ids\" )]\n\tpublic int[] InputTokenIds { get; set; }\n\n\t[JsonPropertyName( \"bos_token_id\" )]\n\tpublic int BosTokenId { get; set; }\n\n\t[JsonPropertyName( \"eos_token_id\" )]\n\tpublic int EosTokenId { get; set; }\n\n\t[JsonPropertyName( \"bos_automatically_added\" )]\n\tpublic bool BosAutomaticallyAdded { get; set; }\n\n\t[JsonPropertyName( \"eos_automatically_added\" )]\n\tpublic bool EosAutomaticallyAdded { get; set; }\n\n\t[JsonPropertyName( \"do_sample\" )]\n\tpublic bool DoSample { get; set; }\n\n\t[JsonPropertyName( \"argmax_tie_break\" )]\n\tpublic string ArgmaxTieBreak { get; set; }\n\n\t[JsonPropertyName( \"logits_processors\" )]\n\tpublic string[] LogitsProcessors { get; set; }\n\n\t[JsonPropertyName( \"raw_logits_used_for_argmax\" )]\n\tpublic bool RawLogitsUsedForArgmax { get; set; }\n\n\t[JsonPropertyName( \"cached_and_uncached_sequences_match\" )]\n\tpublic bool CachedAndUncachedSequencesMatch { get; set; }\n\n\t[JsonPropertyName( \"csharp_baseline_use_cache\" )]\n\tpublic bool CsharpBaselineUseCache { get; set; }\n\n\t[JsonPropertyName( \"max_new_tokens\" )]\n\tpublic int MaxNewTokens { get; set; }\n\n\t[JsonPropertyName( \"generated_token_ids\" )]\n\tpublic int[] GeneratedTokenIds { get; set; }\n\n\t[JsonPropertyName( \"generated_text\" )]\n\tpublic string GeneratedText { get; set; }\n\n\t[JsonPropertyName( \"full_sequence_token_ids\" )]\n\tpublic int[] FullSequenceTokenIds { get; set; }\n\n\t[JsonPropertyName( \"stop_reason\" )]\n\tpublic string StopReason { get; set; }\n\n\t[JsonPropertyName( \"eos_reached\" )]\n\tpublic bool EosReached { get; set; }\n\n\t[JsonPropertyName( \"minimum_top1_top2_margin\" )]\n\tpublic float MinimumTop1Top2Margin { get; set; }\n\n\t[JsonPropertyName( \"minimum_margin_step\" )]\n\tpublic int MinimumMarginStep { get; set; }\n\n\t[JsonPropertyName( \"steps\" )]\n\tpublic GreedyGenerationReferenceStep[] Steps { get; set; }\n\n\t[JsonPropertyName( \"later_full_logit_reference\" )]\n\tpublic GreedyGenerationLogitReference LaterFullLogitReference { get; set; }\n}\n\npublic sealed class GreedyGenerationReferenceStep\n{\n\t[JsonPropertyName( \"step\" )]\n\tpublic int Step { get; set; }\n\n\t[JsonPropertyName( \"input_sequence_length\" )]\n\tpublic int InputSequenceLength { get; set; }\n\n\t[JsonPropertyName( \"input_token_ids\" )]\n\tpublic int[] InputTokenIds { get; set; }\n\n\t[JsonPropertyName( \"new_token_position\" )]\n\tpublic int NewTokenPosition { get; set; }\n\n\t[JsonPropertyName( \"expected_next_token_id\" )]\n\tpublic int ExpectedNextTokenId { get; set; }\n\n\t[JsonPropertyName( \"decoded_token\" )]\n\tpublic string DecodedToken { get; set; }\n\n\t[JsonPropertyName( \"top1_logit\" )]\n\tpublic float Top1Logit { get; set; }\n\n\t[JsonPropertyName( \"top2_token_id\" )]\n\tpublic int Top2TokenId { get; set; }\n\n\t[JsonPropertyName( \"top2_logit\" )]\n\tpublic float Top2Logit { get; set; }\n\n\t[JsonPropertyName( \"top1_top2_margin\" )]\n\tpublic float Top1Top2Margin { get; set; }\n\n\t[JsonPropertyName( \"eos_reached\" )]\n\tpublic bool EosReached { get; set; }\n\n\t[JsonPropertyName( \"top5\" )]\n\tpublic ForwardReferenceTopLogit[] TopFive { get; set; }\n}\n\npublic sealed class GreedyGenerationLogitReference\n{\n\t[JsonPropertyName( \"step\" )]\n\tpublic int Step { get; set; }\n\n\t[JsonPropertyName( \"input_sequence_length\" )]\n\tpublic int InputSequenceLength { get; set; }\n\n\t[JsonPropertyName( \"file\" )]\n\tpublic string File { get; set; }\n\n\t[JsonPropertyName( \"elements\" )]\n\tpublic int Elements { get; set; }\n\n\t[JsonPropertyName( \"bytes\" )]\n\tpublic int Bytes { get; set; }\n\n\t[JsonPropertyName( \"sha256\" )]\n\tpublic string Sha256 { get; set; }\n}\n"
},
{
"Ident": "jeffskitchen.llm_poc",
"Path": "Llm/LlmPaths.cs",
"FileName": "LlmPaths.cs",
"PackageType": "game",
"CodeKind": "Game",
"AssetVersionId": 342521,
"Code": "namespace LlmPoc.Llm;\n\npublic static class LlmPaths\n{\n\tpublic const string RuntimeModelResource = \"models/tinystories.llmmdl\";\n\tpublic const string Root = \"models/tinystories-instruct-1m\";\n\tpublic const string Model = Root + \"/model.bin\";\n\tpublic const string Config = Root + \"/config.json\";\n\tpublic const string Manifest = Root + \"/tensor_manifest.json\";\n\tpublic const string Reference = Root + \"/reference.json\";\n\tpublic const string ReferenceLogits = Root + \"/reference_logits.f32\";\n\tpublic const string ReferenceIntermediates = Root + \"/reference_intermediates.json\";\n\tpublic const string ReferenceEmbedding = Root + \"/reference_embedding.f32\";\n\tpublic const string ReferenceLayer0Ln1 = Root + \"/reference_layer0_ln1.f32\";\n\tpublic const string ReferenceLayer0Q = Root + \"/reference_layer0_q.f32\";\n\tpublic const string ReferenceLayer0K = Root + \"/reference_layer0_k.f32\";\n\tpublic const string ReferenceLayer0V = Root + \"/reference_layer0_v.f32\";\n\tpublic const string ReferenceLayer0QHeads = Root + \"/reference_layer0_q_heads.f32\";\n\tpublic const string ReferenceLayer0KHeads = Root + \"/reference_layer0_k_heads.f32\";\n\tpublic const string ReferenceLayer0VHeads = Root + \"/reference_layer0_v_heads.f32\";\n\tpublic const string ReferenceLayer0ScoresScaledUnmasked =\n\t\tRoot + \"/reference_layer0_scores_scaled_unmasked.f32\";\n\tpublic const string ReferenceLayer0AttentionMask =\n\t\tRoot + \"/reference_layer0_attention_mask.json\";\n\tpublic const string ReferenceLayer0ScoresMaskedPreSoftmax =\n\t\tRoot + \"/reference_layer0_scores_masked_pre_softmax.f32\";\n\tpublic const string ReferenceLayer0AttentionProbs =\n\t\tRoot + \"/reference_layer0_attention_probs.f32\";\n\tpublic const string ReferenceLayer0AttentionContextHeads =\n\t\tRoot + \"/reference_layer0_attention_context_heads.f32\";\n\tpublic const string ReferenceLayer0AttentionMerged =\n\t\tRoot + \"/reference_layer0_attention_merged.f32\";\n\tpublic const string ReferenceLayer0AttentionOutProj =\n\t\tRoot + \"/reference_layer0_attention_out_proj.f32\";\n\tpublic const string ReferenceLayer0AttentionResidual =\n\t\tRoot + \"/reference_layer0_attention_residual.f32\";\n\tpublic const string ReferenceLayer0Ln2 = Root + \"/reference_layer0_ln2.f32\";\n\tpublic const string ReferenceLayer0MlpFc = Root + \"/reference_layer0_mlp_fc.f32\";\n\tpublic const string ReferenceLayer0MlpGelu = Root + \"/reference_layer0_mlp_gelu.f32\";\n\tpublic const string ReferenceLayer0MlpProj = Root + \"/reference_layer0_mlp_proj.f32\";\n\tpublic const string ReferenceLayer0Output = Root + \"/reference_layer0_output.f32\";\n\tpublic const string ReferenceFinalLayerNorm = Root + \"/reference_final_ln.f32\";\n\tpublic const string ReferenceFinalLogitsLastPosition =\n\t\tRoot + \"/reference_final_logits_last_position.f32\";\n\tpublic const string ReferenceGreedyGeneration =\n\t\tRoot + \"/reference_greedy_generation.json\";\n\tpublic const string ReferenceGenerationStep11Logits =\n\t\tRoot + \"/reference_generation_step11_logits.f32\";\n\tpublic const string ReferenceLayer1LocalMaskLen260 =\n\t\tRoot + \"/reference_layer1_local_mask_len260.bin\";\n\tpublic const string ReferenceLayer1LocalMaskLen260Metadata =\n\t\tRoot + \"/reference_layer1_local_mask_len260.json\";\n\tpublic const string Tokenizer = Root + \"/tokenizer/tokenizer.json\";\n\tpublic const string TokenizerConfig = Root + \"/tokenizer/tokenizer_config.json\";\n\n\tpublic static string ReferenceStage( string fileName )\n\t{\n\t\tif ( string.IsNullOrWhiteSpace( fileName ) || fileName.Contains( '/' ) ||\n\t\t\tfileName.Contains( '\\\\' ) || fileName.Contains( \"..\" ) )\n\t\t{\n\t\t\tthrow new ArgumentException(\n\t\t\t\t$\"[LLM:ERROR] Reference stage filename '{fileName}' is invalid.\",\n\t\t\t\tnameof( fileName ) );\n\t\t}\n\t\treturn Root + \"/\" + fileName;\n\t}\n}\n"
},
{
"Ident": "jeffskitchen.llm_poc",
"Path": ".obj/__compiler_extra.cs",
"FileName": "__compiler_extra.cs",
"PackageType": "game",
"CodeKind": "Game",
"AssetVersionId": 342521,
"Code": "global using static Sandbox.Internal.GlobalGameNamespace;\r\nglobal using Microsoft.AspNetCore.Components;\r\nglobal using Microsoft.AspNetCore.Components.Rendering;\r\n[assembly: global::System.Reflection.AssemblyMetadata( \"AddonTitle\", \"LLM POC\" )]\r\n[assembly: global::System.Reflection.AssemblyMetadata( \"AddonIdent\", \"llm_poc\" )]\r\n[assembly: global::System.Reflection.AssemblyMetadata( \"OrgIdent\", \"jeffskitchen\" )]\r\n[assembly: global::System.Reflection.AssemblyMetadata( \"Ident\", \"jeffskitchen.llm_poc\" )]\r\n[assembly: global::System.Reflection.AssemblyMetadata( \"EngineVersion\", \"28\" )]\r\n[assembly: global::System.Reflection.AssemblyMetadata( \"EngineMinorVersion\", \"1\" )]\r\n\r\n[assembly: System.Runtime.Versioning.TargetFramework( \".NETCoreApp,Version=v9.0\", FrameworkDisplayName = \".NET 9.0\" )]\r\n[assembly: global::System.Reflection.AssemblyMetadata( \"CompileTime\", \"2026-08-17T04:57:47.5628694Z\" )]\r\n[assembly: global::System.Reflection.AssemblyVersion(\"0.0.124.0\")]\r\n[assembly: global::System.Reflection.AssemblyFileVersion(\"0.0.124.0\")]"
},
{
"Ident": "jeffskitchen.llm_poc",
"Path": "Assembly.cs",
"FileName": "Assembly.cs",
"PackageType": "game",
"CodeKind": "Game",
"AssetVersionId": 342521,
"Code": "global using Sandbox;\nglobal using System;\nglobal using System.Collections.Generic;\nglobal using System.Linq;\nglobal using System.Threading.Tasks;\n"
},
{
"Ident": "jeffskitchen.llm_poc",
"Path": "Llm/GreedyGenerationParity.cs",
"FileName": "GreedyGenerationParity.cs",
"PackageType": "game",
"CodeKind": "Game",
"AssetVersionId": 342521,
"Code": "using Sandbox.Diagnostics;\n\nnamespace LlmPoc.Llm;\n\npublic sealed class GreedyGenerationParityResult\n{\n\tpublic GreedyGenerationResult Generation { get; init; }\n\tpublic NumericComparison LaterStepLogitComparison { get; init; }\n\tpublic int LaterStepLogitIndex { get; init; }\n\tpublic float MinimumMargin { get; init; }\n\tpublic int MinimumMarginStep { get; init; }\n\tpublic double ValidationHarnessMilliseconds { get; init; }\n}\n\npublic static class GreedyGenerationParity\n{\n\tprivate const double LogitAbsoluteTolerance = 5.0e-5;\n\tprivate const double LogitRelativeTolerance = 1.0e-5;\n\tprivate const double TopLogitAbsoluteTolerance = 5.0e-5;\n\n\tpublic static GreedyGenerationParityResult Validate(\n\t\tSboxLlmModel model,\n\t\tTinyStoriesConfig config,\n\t\tGpt2ByteBpeTokenizer tokenizer,\n\t\tLlmReferenceData historicalReference,\n\t\tGreedyGenerationReferenceDocument reference )\n\t{\n\t\tif ( model is null ) throw new ArgumentNullException( nameof( model ) );\n\t\tif ( config is null ) throw new ArgumentNullException( nameof( config ) );\n\t\tif ( tokenizer is null ) throw new ArgumentNullException( nameof( tokenizer ) );\n\t\tif ( historicalReference is null )\n\t\t\tthrow new ArgumentNullException( nameof( historicalReference ) );\n\t\tif ( reference is null ) throw new ArgumentNullException( nameof( reference ) );\n\n\t\tValidateReference( config, tokenizer, historicalReference, reference );\n\t\tfloat[] laterExpected = ReferenceFloatData.LoadFromMounted(\n\t\t\tLlmPaths.ReferenceStage( reference.LaterFullLogitReference.File ),\n\t\t\treference.LaterFullLogitReference.Elements );\n\t\tNumericComparison laterComparison = null;\n\t\tFastTimer validationTimer = FastTimer.StartNew();\n\n\t\tGreedyGenerationResult generation = TinyStoriesGreedyGenerator.Generate(\n\t\t\tmodel,\n\t\t\tconfig,\n\t\t\ttokenizer,\n\t\t\treference.InputTokenIds,\n\t\t\treference.MaxNewTokens,\n\t\t\tobservation =>\n\t\t\t{\n\t\t\t\tGreedyGenerationStepResult actual = observation.Step;\n\t\t\t\tif ( actual.Step < 0 || actual.Step >= reference.Steps.Length )\n\t\t\t\t{\n\t\t\t\t\tthrow new InvalidOperationException(\n\t\t\t\t\t\t$\"[LLM:ERROR] Generation produced unexpected step {actual.Step}.\" );\n\t\t\t\t}\n\t\t\t\tGreedyGenerationReferenceStep expected = reference.Steps[actual.Step];\n\t\t\t\tValidateStepContext( expected, observation.InputTokenIds );\n\t\t\t\tbool tokenPassed = actual.TokenId == expected.ExpectedNextTokenId;\n\t\t\t\tif ( !tokenPassed )\n\t\t\t\t{\n\t\t\t\t\tLogMismatch( tokenizer, expected, observation );\n\t\t\t\t\tthrow new InvalidOperationException(\n\t\t\t\t\t\t$\"[LLM:ERROR] Greedy generation first mismatch at step {actual.Step}: \" +\n\t\t\t\t\t\t$\"expected token {expected.ExpectedNextTokenId}, actual {actual.TokenId}. \" +\n\t\t\t\t\t\t\"No mismatching token was appended.\" );\n\t\t\t\t}\n\n\t\t\t\tif ( actual.DecodedToken != expected.DecodedToken )\n\t\t\t\t{\n\t\t\t\t\tthrow new InvalidOperationException(\n\t\t\t\t\t\t$\"[LLM:ERROR] Generation step {actual.Step} token {actual.TokenId} decoded \" +\n\t\t\t\t\t\t$\"as '{EscapeVisible( actual.DecodedToken )}', Python expected \" +\n\t\t\t\t\t\t$\"'{EscapeVisible( expected.DecodedToken )}'.\" );\n\t\t\t\t}\n\t\t\t\tValidateTopFive( tokenizer, expected, actual );\n\t\t\t\tif ( actual.EosReached != expected.EosReached )\n\t\t\t\t{\n\t\t\t\t\tthrow new InvalidOperationException(\n\t\t\t\t\t\t$\"[LLM:ERROR] Generation step {actual.Step} EOS state expected \" +\n\t\t\t\t\t\t$\"{expected.EosReached}, actual {actual.EosReached}.\" );\n\t\t\t\t}\n\n\t\t\t\tif ( actual.Step == reference.LaterFullLogitReference.Step )\n\t\t\t\t{\n\t\t\t\t\tlaterComparison = TensorDiagnostics.Compare(\n\t\t\t\t\t\tlaterExpected,\n\t\t\t\t\t\tobservation.Forward.Logits.Data,\n\t\t\t\t\t\tLogitAbsoluteTolerance,\n\t\t\t\t\t\tLogitRelativeTolerance );\n\t\t\t\t\tTensorSummary summary = TensorDiagnostics.Summarize(\n\t\t\t\t\t\tobservation.Forward.Logits );\n\t\t\t\t\tLlmLog.Info(\n\t\t\t\t\t\t\"PARITY\",\n\t\t\t\t\t\t$\"generation.step{actual.Step}.logits count={summary.ElementCount:N0} \" +\n\t\t\t\t\t\t$\"finite={summary.FiniteCount:N0}/{summary.ElementCount:N0} \" +\n\t\t\t\t\t\t$\"maxAbs={laterComparison.MaximumAbsoluteError:G12} \" +\n\t\t\t\t\t\t$\"meanAbs={laterComparison.MeanAbsoluteError:G12} \" +\n\t\t\t\t\t\t$\"maxRel={laterComparison.MaximumRelativeError:G12} \" +\n\t\t\t\t\t\t$\"worst_vocab={laterComparison.MaximumErrorIndex} \" +\n\t\t\t\t\t\t$\"expected={laterComparison.ExpectedAtMaximumError:G9} \" +\n\t\t\t\t\t\t$\"actual={laterComparison.ActualAtMaximumError:G9} \" +\n\t\t\t\t\t\t$\"absTol={LogitAbsoluteTolerance:G1} relTol={LogitRelativeTolerance:G1} \" +\n\t\t\t\t\t\t$\"{(laterComparison.Passed ? \"PASS\" : \"FAIL\")}\" );\n\t\t\t\t\tif ( !laterComparison.Passed )\n\t\t\t\t\t{\n\t\t\t\t\t\tthrow new InvalidOperationException(\n\t\t\t\t\t\t\t$\"[LLM:ERROR] Later generation step {actual.Step} full-logit parity \" +\n\t\t\t\t\t\t\t$\"failed: {laterComparison}.\" );\n\t\t\t\t\t}\n\t\t\t\t}\n\n\t\t\t\tLlmLog.Info(\n\t\t\t\t\t\"GEN\",\n\t\t\t\t\t$\"step={actual.Step} context={actual.InputSequenceLength} \" +\n\t\t\t\t\t$\"position={actual.NewTokenPosition} expected={expected.ExpectedNextTokenId} \" +\n\t\t\t\t\t$\"actual={actual.TokenId} token='{EscapeVisible( actual.DecodedToken )}' \" +\n\t\t\t\t\t$\"margin={actual.Top1Top2Margin:G9} forward_ms={actual.ForwardMilliseconds:N4} \" +\n\t\t\t\t\t$\"lm_head_ms={actual.LmHeadMilliseconds:N4} PASS\" );\n\t\t\t} );\n\n\t\tRequireExactArray(\n\t\t\t\"generated token IDs\", reference.GeneratedTokenIds, generation.GeneratedTokenIds );\n\t\tRequireExactArray(\n\t\t\t\"full generated sequence\", reference.FullSequenceTokenIds,\n\t\t\tgeneration.FullSequenceTokenIds );\n\t\tif ( generation.GeneratedText != reference.GeneratedText )\n\t\t{\n\t\t\tthrow new InvalidOperationException(\n\t\t\t\t$\"[LLM:ERROR] Generated text expected '{EscapeVisible( reference.GeneratedText )}', \" +\n\t\t\t\t$\"actual '{EscapeVisible( generation.GeneratedText )}'.\" );\n\t\t}\n\t\tif ( generation.StopReason != reference.StopReason ||\n\t\t\tgeneration.EosReached != reference.EosReached )\n\t\t{\n\t\t\tthrow new InvalidOperationException(\n\t\t\t\t$\"[LLM:ERROR] Generation stop expected reason={reference.StopReason} \" +\n\t\t\t\t$\"eos={reference.EosReached}, actual reason={generation.StopReason} \" +\n\t\t\t\t$\"eos={generation.EosReached}.\" );\n\t\t}\n\t\tif ( laterComparison is null )\n\t\t{\n\t\t\tthrow new InvalidOperationException(\n\t\t\t\t$\"[LLM:ERROR] Later-step full-logit checkpoint \" +\n\t\t\t\t$\"{reference.LaterFullLogitReference.Step} did not execute.\" );\n\t\t}\n\n\t\tGreedyGenerationStepResult minimum = generation.Steps[0];\n\t\tfor ( int index = 1; index < generation.Steps.Length; index++ )\n\t\t{\n\t\t\tif ( generation.Steps[index].Top1Top2Margin < minimum.Top1Top2Margin )\n\t\t\t{\n\t\t\t\tminimum = generation.Steps[index];\n\t\t\t}\n\t\t}\n\t\tif ( minimum.Step != reference.MinimumMarginStep ||\n\t\t\tMath.Abs( minimum.Top1Top2Margin - reference.MinimumTop1Top2Margin ) >\n\t\t\t\tTopLogitAbsoluteTolerance )\n\t\t{\n\t\t\tthrow new InvalidOperationException(\n\t\t\t\t$\"[LLM:ERROR] Minimum generation margin expected step=\" +\n\t\t\t\t$\"{reference.MinimumMarginStep} value={reference.MinimumTop1Top2Margin:G9}, \" +\n\t\t\t\t$\"actual step={minimum.Step} value={minimum.Top1Top2Margin:G9}.\" );\n\t\t}\n\n\t\tLlmLog.Info(\n\t\t\t\"PARITY\",\n\t\t\t$\"greedy_sequence generated={generation.GeneratedTokenIds.Length} \" +\n\t\t\t$\"matched={generation.GeneratedTokenIds.Length} ids=\" +\n\t\t\t$\"[{string.Join( \",\", generation.GeneratedTokenIds )}] \" +\n\t\t\t$\"text='{EscapeVisible( generation.GeneratedText )}' stop={generation.StopReason} PASS\" );\n\t\tLlmLog.Info(\n\t\t\t\"GEN\",\n\t\t\t$\"minimum_margin={minimum.Top1Top2Margin:G9} step={minimum.Step} \" +\n\t\t\t$\"first_token_ms={generation.Steps[0].StepMilliseconds:N4} \" +\n\t\t\t$\"last_token_ms={generation.Steps[^1].StepMilliseconds:N4} \" +\n\t\t\t$\"total_forward_ms={generation.TotalForwardMilliseconds:N4} \" +\n\t\t\t$\"total_generation_ms={generation.TotalGenerationMilliseconds:N4} \" +\n\t\t\t$\"average_ms_per_token=\" +\n\t\t\t$\"{generation.TotalGenerationMilliseconds / generation.Steps.Length:N4} \" +\n\t\t\t$\"tokens_per_second=\" +\n\t\t\t$\"{generation.Steps.Length * 1000.0 / generation.TotalGenerationMilliseconds:N4} \" +\n\t\t\t\"kv_cache=false PASS\" );\n\n\t\treturn new GreedyGenerationParityResult\n\t\t{\n\t\t\tGeneration = generation,\n\t\t\tLaterStepLogitComparison = laterComparison,\n\t\t\tLaterStepLogitIndex = reference.LaterFullLogitReference.Step,\n\t\t\tMinimumMargin = minimum.Top1Top2Margin,\n\t\t\tMinimumMarginStep = minimum.Step,\n\t\t\tValidationHarnessMilliseconds = validationTimer.ElapsedMilliSeconds\n\t\t};\n\t}\n\n\tprivate static void ValidateReference(\n\t\tTinyStoriesConfig config,\n\t\tGpt2ByteBpeTokenizer tokenizer,\n\t\tLlmReferenceData historical,\n\t\tGreedyGenerationReferenceDocument reference )\n\t{\n\t\tRequireExactArray( \"reference prompt IDs\", historical.InputTokenIds, reference.InputTokenIds );\n\t\tRequireExactArray(\n\t\t\t\"historical generated IDs\", historical.GeneratedTokenIds,\n\t\t\treference.GeneratedTokenIds );\n\t\tif ( historical.Prompt != reference.Prompt ||\n\t\t\thistorical.GeneratedText != reference.GeneratedText )\n\t\t{\n\t\t\tthrow new InvalidOperationException(\n\t\t\t\t\"[LLM:ERROR] Compact generation reference differs from historical reference.json.\" );\n\t\t}\n\t\tif ( reference.EosTokenId != config.EosTokenId ||\n\t\t\treference.BosTokenId != config.BosTokenId )\n\t\t{\n\t\t\tthrow new InvalidOperationException(\n\t\t\t\t$\"[LLM:ERROR] Generation special IDs expected BOS/EOS \" +\n\t\t\t\t$\"{config.BosTokenId}/{config.EosTokenId}, found \" +\n\t\t\t\t$\"{reference.BosTokenId}/{reference.EosTokenId}.\" );\n\t\t}\n\t\tif ( reference.DoSample || reference.BosAutomaticallyAdded ||\n\t\t\treference.EosAutomaticallyAdded || !reference.RawLogitsUsedForArgmax ||\n\t\t\t!reference.CachedAndUncachedSequencesMatch || reference.CsharpBaselineUseCache ||\n\t\t\treference.LogitsProcessors is null || reference.LogitsProcessors.Length != 0 ||\n\t\t\treference.ArgmaxTieBreak != \"first (lowest) vocabulary index\" )\n\t\t{\n\t\t\tthrow new InvalidOperationException(\n\t\t\t\t\"[LLM:ERROR] Generation reference does not describe raw, uncached, \" +\n\t\t\t\t\"deterministic first-index argmax semantics.\" );\n\t\t}\n\t\tif ( reference.MaxNewTokens <= 0 || reference.Steps is null ||\n\t\t\treference.Steps.Length != reference.GeneratedTokenIds.Length ||\n\t\t\treference.Steps.Length > reference.MaxNewTokens )\n\t\t{\n\t\t\tthrow new InvalidOperationException(\n\t\t\t\t\"[LLM:ERROR] Generation reference step/token counts are inconsistent.\" );\n\t\t}\n\t\tif ( tokenizer.Decode( reference.GeneratedTokenIds ) != reference.GeneratedText )\n\t\t{\n\t\t\tthrow new InvalidOperationException(\n\t\t\t\t\"[LLM:ERROR] C# tokenizer cannot reproduce Python generated text from \" +\n\t\t\t\t\"the authoritative token sequence.\" );\n\t\t}\n\t\tif ( reference.LaterFullLogitReference is null ||\n\t\t\treference.LaterFullLogitReference.Step < 1 ||\n\t\t\treference.LaterFullLogitReference.Step >= reference.Steps.Length ||\n\t\t\treference.LaterFullLogitReference.Elements != config.VocabularySize ||\n\t\t\treference.LaterFullLogitReference.Bytes != config.VocabularySize * sizeof( float ) )\n\t\t{\n\t\t\tthrow new InvalidOperationException(\n\t\t\t\t\"[LLM:ERROR] Later generation full-logit reference metadata is invalid.\" );\n\t\t}\n\t}\n\n\tprivate static void ValidateStepContext(\n\t\tGreedyGenerationReferenceStep expected,\n\t\tint[] actualInput )\n\t{\n\t\tif ( expected.Step < 0 || expected.InputTokenIds is null ||\n\t\t\texpected.InputSequenceLength != expected.InputTokenIds.Length ||\n\t\t\texpected.NewTokenPosition != expected.InputSequenceLength )\n\t\t{\n\t\t\tthrow new InvalidOperationException(\n\t\t\t\t$\"[LLM:ERROR] Generation reference step {expected.Step} context metadata is invalid.\" );\n\t\t}\n\t\tRequireExactArray( $\"generation step {expected.Step} input\", expected.InputTokenIds, actualInput );\n\t}\n\n\tprivate static void ValidateTopFive(\n\t\tGpt2ByteBpeTokenizer tokenizer,\n\t\tGreedyGenerationReferenceStep expected,\n\t\tGreedyGenerationStepResult actual )\n\t{\n\t\tif ( expected.TopFive is null || expected.TopFive.Length != 5 ||\n\t\t\tactual.TopFive is null || actual.TopFive.Length != 5 )\n\t\t{\n\t\t\tthrow new InvalidOperationException(\n\t\t\t\t$\"[LLM:ERROR] Generation step {actual.Step} requires five top-logit entries.\" );\n\t\t}\n\t\tfor ( int rank = 0; rank < 5; rank++ )\n\t\t{\n\t\t\tForwardReferenceTopLogit expectedRank = expected.TopFive[rank];\n\t\t\tLogitRank actualRank = actual.TopFive[rank];\n\t\t\tstring decoded = tokenizer.Decode( new[] { actualRank.TokenId } );\n\t\t\tif ( expectedRank.Rank != actualRank.Rank ||\n\t\t\t\texpectedRank.TokenId != actualRank.TokenId ||\n\t\t\t\texpectedRank.DecodedToken != decoded ||\n\t\t\t\tMath.Abs( expectedRank.Logit - actualRank.Logit ) > TopLogitAbsoluteTolerance )\n\t\t\t{\n\t\t\t\tthrow new InvalidOperationException(\n\t\t\t\t\t$\"[LLM:ERROR] Generation step {actual.Step} top-5 rank {rank + 1} \" +\n\t\t\t\t\t$\"expected token={expectedRank.TokenId} logit={expectedRank.Logit:G9} \" +\n\t\t\t\t\t$\"decoded='{EscapeVisible( expectedRank.DecodedToken )}', actual \" +\n\t\t\t\t\t$\"token={actualRank.TokenId} logit={actualRank.Logit:G9} \" +\n\t\t\t\t\t$\"decoded='{EscapeVisible( decoded )}'.\" );\n\t\t\t}\n\t\t}\n\t}\n\n\tprivate static void LogMismatch(\n\t\tGpt2ByteBpeTokenizer tokenizer,\n\t\tGreedyGenerationReferenceStep expected,\n\t\tGreedyGenerationStepObservation observation )\n\t{\n\t\tGreedyGenerationStepResult actual = observation.Step;\n\t\tLlmLog.Error(\n\t\t\t$\"Greedy mismatch step={actual.Step} context={actual.InputSequenceLength} \" +\n\t\t\t$\"input=[{string.Join( \",\", observation.InputTokenIds )}] \" +\n\t\t\t$\"expected={expected.ExpectedNextTokenId} \" +\n\t\t\t$\"expected_piece='{EscapeVisible( expected.DecodedToken )}' actual={actual.TokenId} \" +\n\t\t\t$\"actual_piece='{EscapeVisible( tokenizer.Decode( new[] { actual.TokenId } ) )}' \" +\n\t\t\t$\"expected_margin={expected.Top1Top2Margin:G9} \" +\n\t\t\t$\"actual_margin={actual.Top1Top2Margin:G9}.\" );\n\t\tLlmLog.Error(\n\t\t\t$\"Python top5=[{string.Join( \",\", expected.TopFive.Select( item => $\"{item.TokenId}:{item.Logit:G9}\" ) )}] \" +\n\t\t\t$\"C# top5=[{string.Join( \",\", actual.TopFive.Select( item => $\"{item.TokenId}:{item.Logit:G9}\" ) )}]. \" +\n\t\t\t\"Generate a full Python logit reference for this step only before changing tolerances.\" );\n\t}\n\n\tprivate static void RequireExactArray( string label, int[] expected, int[] actual )\n\t{\n\t\tif ( expected is null || actual is null || expected.Length != actual.Length )\n\t\t{\n\t\t\tthrow new InvalidOperationException(\n\t\t\t\t$\"[LLM:ERROR] {label} expected length {expected?.Length ?? -1}, \" +\n\t\t\t\t$\"actual {actual?.Length ?? -1}.\" );\n\t\t}\n\t\tfor ( int index = 0; index < expected.Length; index++ )\n\t\t{\n\t\t\tif ( expected[index] != actual[index] )\n\t\t\t{\n\t\t\t\tthrow new InvalidOperationException(\n\t\t\t\t\t$\"[LLM:ERROR] {label} mismatch at index {index}: \" +\n\t\t\t\t\t$\"expected={expected[index]}, actual={actual[index]}.\" );\n\t\t\t}\n\t\t}\n\t}\n\n\tprivate static string EscapeVisible( string value )\n\t{\n\t\treturn (value ?? \"<null>\")\n\t\t\t.Replace( \"\\\\\", \"\\\\\\\\\" )\n\t\t\t.Replace( \"\\r\", \"\\\\r\" )\n\t\t\t.Replace( \"\\n\", \"\\\\n\" )\n\t\t\t.Replace( \"\\t\", \"\\\\t\" )\n\t\t\t.Replace( \"'\", \"\\\\'\" );\n\t}\n}\n"
},
{
"Ident": "jeffskitchen.llm_poc",
"Path": "Llm/LlmLog.cs",
"FileName": "LlmLog.cs",
"PackageType": "game",
"CodeKind": "Game",
"AssetVersionId": 342521,
"Code": "namespace LlmPoc.Llm;\n\npublic static class LlmLog\n{\n\tpublic static bool TraceEnabled { get; set; }\n\n\tpublic static void Info( string category, string message )\n\t{\n\t\tLog.Info( $\"[LLM:{category}] {message}\" );\n\t}\n\n\tpublic static void Trace( string category, string message )\n\t{\n\t\tif ( TraceEnabled )\n\t\t{\n\t\t\tLog.Info( $\"[LLM:{category}] {message}\" );\n\t\t}\n\t}\n\n\tpublic static void Warning( string category, string message )\n\t{\n\t\tLog.Warning( $\"[LLM:{category}] {message}\" );\n\t}\n\n\tpublic static void Error( string message )\n\t{\n\t\tLog.Error( $\"[LLM:ERROR] {message}\" );\n\t}\n}\n"
},
{
"Ident": "jeffskitchen.llm_poc",
"Path": "Llm/TensorDiagnostics.cs",
"FileName": "TensorDiagnostics.cs",
"PackageType": "game",
"CodeKind": "Game",
"AssetVersionId": 342521,
"Code": "using System.Text;\n\nnamespace LlmPoc.Llm;\n\npublic sealed class TensorSummary\n{\n\tpublic string Name { get; init; }\n\tpublic string Shape { get; init; }\n\tpublic long ElementCount { get; init; }\n\tpublic float Minimum { get; init; }\n\tpublic float Maximum { get; init; }\n\tpublic double Mean { get; init; }\n\tpublic double StandardDeviation { get; init; }\n\tpublic double Rms { get; init; }\n\tpublic int NaNCount { get; init; }\n\tpublic int PositiveInfinityCount { get; init; }\n\tpublic int NegativeInfinityCount { get; init; }\n\tpublic string FirstValues { get; init; }\n\tpublic string LastValues { get; init; }\n\tpublic ulong Checksum { get; init; }\n\n\tpublic bool IsFinite => NaNCount == 0 && PositiveInfinityCount == 0 && NegativeInfinityCount == 0;\n\tpublic long FiniteCount => ElementCount - NaNCount - PositiveInfinityCount - NegativeInfinityCount;\n\n\tpublic override string ToString()\n\t{\n\t\treturn $\"name={Name} shape={Shape} count={ElementCount:N0} \" +\n\t\t\t$\"min={Minimum:G9} max={Maximum:G9} mean={Mean:G12} \" +\n\t\t\t$\"std={StandardDeviation:G12} rms={Rms:G12} finite={FiniteCount:N0}/{ElementCount:N0} \" +\n\t\t\t$\"nan={NaNCount} +inf={PositiveInfinityCount} -inf={NegativeInfinityCount} \" +\n\t\t\t$\"first={FirstValues} last={LastValues} fnv1a64={Checksum:X16}\";\n\t}\n}\n\npublic sealed class NumericComparison\n{\n\tpublic int ElementCount { get; init; }\n\tpublic double MaximumAbsoluteError { get; init; }\n\tpublic double MeanAbsoluteError { get; init; }\n\tpublic double MaximumRelativeError { get; init; }\n\tpublic int MaximumErrorIndex { get; init; }\n\tpublic float ExpectedAtMaximumError { get; init; }\n\tpublic float ActualAtMaximumError { get; init; }\n\tpublic int FirstFailingIndex { get; init; }\n\tpublic float ExpectedAtFirstFailure { get; init; }\n\tpublic float ActualAtFirstFailure { get; init; }\n\tpublic double AbsoluteTolerance { get; init; }\n\tpublic double RelativeTolerance { get; init; }\n\tpublic bool Passed { get; init; }\n\n\tpublic override string ToString()\n\t{\n\t\tstring failure = FirstFailingIndex < 0\n\t\t\t? \"\"\n\t\t\t: $\" first_fail_index={FirstFailingIndex} \" +\n\t\t\t\t$\"first_expected={ExpectedAtFirstFailure:G9} first_actual={ActualAtFirstFailure:G9}\";\n\t\treturn $\"count={ElementCount:N0} max_abs={MaximumAbsoluteError:G12} \" +\n\t\t\t$\"mean_abs={MeanAbsoluteError:G12} max_rel={MaximumRelativeError:G12} \" +\n\t\t\t$\"max_index={MaximumErrorIndex} expected={ExpectedAtMaximumError:G9} \" +\n\t\t\t$\"actual={ActualAtMaximumError:G9} abs_tol={AbsoluteTolerance:G6} \" +\n\t\t\t$\"rel_tol={RelativeTolerance:G6}{failure} {(Passed ? \"PASS\" : \"FAIL\")}\";\n\t}\n}\n\npublic static class TensorDiagnostics\n{\n\tprivate const ulong FnvOffsetBasis = 14695981039346656037UL;\n\tprivate const ulong FnvPrime = 1099511628211UL;\n\n\tpublic static TensorSummary Summarize( Tensor tensor, int edgeCount = 4 )\n\t{\n\t\tif ( tensor is null )\n\t\t{\n\t\t\tthrow new ArgumentNullException( nameof( tensor ) );\n\t\t}\n\t\treturn Summarize( tensor.Name, tensor.ShapeText, tensor.Data, edgeCount );\n\t}\n\n\tpublic static TensorSummary Summarize(\n\t\tstring name,\n\t\tstring shape,\n\t\tReadOnlySpan<float> values,\n\t\tint edgeCount = 4 )\n\t{\n\t\tif ( edgeCount < 0 || edgeCount > 32 )\n\t\t{\n\t\t\tthrow new ArgumentOutOfRangeException(\n\t\t\t\tnameof( edgeCount ), edgeCount, \"[LLM:ERROR] Edge count must be between 0 and 32.\" );\n\t\t}\n\t\tif ( values.Length == 0 )\n\t\t{\n\t\t\tthrow new ArgumentException( $\"[LLM:ERROR] {name} cannot be summarized because it is empty.\" );\n\t\t}\n\n\t\tfloat minimum = float.PositiveInfinity;\n\t\tfloat maximum = float.NegativeInfinity;\n\t\tdouble sum = 0;\n\t\tdouble sumSquares = 0;\n\t\tint finiteCount = 0;\n\t\tint nanCount = 0;\n\t\tint positiveInfinityCount = 0;\n\t\tint negativeInfinityCount = 0;\n\t\tulong checksum = FnvOffsetBasis;\n\n\t\tfor ( int index = 0; index < values.Length; index++ )\n\t\t{\n\t\t\tfloat value = values[index];\n\t\t\tuint bits = unchecked( (uint)BitConverter.SingleToInt32Bits( value ) );\n\t\t\tchecksum = HashByte( checksum, (byte)bits );\n\t\t\tchecksum = HashByte( checksum, (byte)(bits >> 8) );\n\t\t\tchecksum = HashByte( checksum, (byte)(bits >> 16) );\n\t\t\tchecksum = HashByte( checksum, (byte)(bits >> 24) );\n\n\t\t\tif ( float.IsNaN( value ) )\n\t\t\t{\n\t\t\t\tnanCount++;\n\t\t\t\tcontinue;\n\t\t\t}\n\t\t\tif ( float.IsPositiveInfinity( value ) )\n\t\t\t{\n\t\t\t\tpositiveInfinityCount++;\n\t\t\t\tcontinue;\n\t\t\t}\n\t\t\tif ( float.IsNegativeInfinity( value ) )\n\t\t\t{\n\t\t\t\tnegativeInfinityCount++;\n\t\t\t\tcontinue;\n\t\t\t}\n\n\t\t\tminimum = Math.Min( minimum, value );\n\t\t\tmaximum = Math.Max( maximum, value );\n\t\t\tsum += value;\n\t\t\tsumSquares += (double)value * value;\n\t\t\tfiniteCount++;\n\t\t}\n\n\t\tdouble mean = finiteCount == 0 ? double.NaN : sum / finiteCount;\n\t\tdouble rms = finiteCount == 0 ? double.NaN : Math.Sqrt( sumSquares / finiteCount );\n\t\tdouble variance = finiteCount == 0 ? double.NaN : Math.Max( 0, sumSquares / finiteCount - mean * mean );\n\n\t\treturn new TensorSummary\n\t\t{\n\t\t\tName = name,\n\t\t\tShape = shape,\n\t\t\tElementCount = values.Length,\n\t\t\tMinimum = finiteCount == 0 ? float.NaN : minimum,\n\t\t\tMaximum = finiteCount == 0 ? float.NaN : maximum,\n\t\t\tMean = mean,\n\t\t\tStandardDeviation = Math.Sqrt( variance ),\n\t\t\tRms = rms,\n\t\t\tNaNCount = nanCount,\n\t\t\tPositiveInfinityCount = positiveInfinityCount,\n\t\t\tNegativeInfinityCount = negativeInfinityCount,\n\t\t\tFirstValues = FormatEdge( values, 0, Math.Min( edgeCount, values.Length ) ),\n\t\t\tLastValues = FormatEdge( values, Math.Max( 0, values.Length - edgeCount ), Math.Min( edgeCount, values.Length ) ),\n\t\t\tChecksum = checksum\n\t\t};\n\t}\n\n\tpublic static NumericComparison Compare(\n\t\tReadOnlySpan<float> expected,\n\t\tReadOnlySpan<float> actual,\n\t\tdouble absoluteTolerance = 1e-4,\n\t\tdouble relativeTolerance = 1e-3 )\n\t{\n\t\tif ( expected.Length != actual.Length )\n\t\t{\n\t\t\tthrow new ArgumentException(\n\t\t\t\t$\"[LLM:ERROR] Parity comparison shape mismatch: expected {expected.Length:N0} \" +\n\t\t\t\t$\"elements, actual {actual.Length:N0}.\" );\n\t\t}\n\t\tif ( expected.Length == 0 )\n\t\t{\n\t\t\tthrow new ArgumentException( \"[LLM:ERROR] Parity comparison cannot use empty arrays.\" );\n\t\t}\n\t\tif ( absoluteTolerance < 0 || relativeTolerance < 0 )\n\t\t{\n\t\t\tthrow new ArgumentOutOfRangeException(\n\t\t\t\tnameof( absoluteTolerance ), \"[LLM:ERROR] Parity tolerances cannot be negative.\" );\n\t\t}\n\n\t\tdouble maxAbsolute = -1;\n\t\tdouble maxRelative = 0;\n\t\tdouble absoluteSum = 0;\n\t\tint maxIndex = 0;\n\t\tint firstFailingIndex = -1;\n\t\tbool passed = true;\n\n\t\tfor ( int index = 0; index < expected.Length; index++ )\n\t\t{\n\t\t\tfloat expectedValue = expected[index];\n\t\t\tfloat actualValue = actual[index];\n\t\t\tbool finite = float.IsFinite( expectedValue ) && float.IsFinite( actualValue );\n\t\t\tif ( !finite )\n\t\t\t{\n\t\t\t\tpassed = false;\n\t\t\t\tif ( firstFailingIndex < 0 )\n\t\t\t\t{\n\t\t\t\t\tfirstFailingIndex = index;\n\t\t\t\t}\n\t\t\t}\n\n\t\t\tdouble absolute = Math.Abs( (double)actualValue - expectedValue );\n\t\t\tdouble scale = Math.Max( Math.Abs( expectedValue ), 1e-12 );\n\t\t\tdouble relative = absolute / scale;\n\t\t\tabsoluteSum += absolute;\n\t\t\tmaxRelative = Math.Max( maxRelative, relative );\n\t\t\tif ( absolute > maxAbsolute )\n\t\t\t{\n\t\t\t\tmaxAbsolute = absolute;\n\t\t\t\tmaxIndex = index;\n\t\t\t}\n\n\t\t\tif ( absolute > absoluteTolerance + relativeTolerance * Math.Abs( expectedValue ) )\n\t\t\t{\n\t\t\t\tpassed = false;\n\t\t\t\tif ( firstFailingIndex < 0 )\n\t\t\t\t{\n\t\t\t\t\tfirstFailingIndex = index;\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\n\t\treturn new NumericComparison\n\t\t{\n\t\t\tElementCount = expected.Length,\n\t\t\tMaximumAbsoluteError = maxAbsolute,\n\t\t\tMeanAbsoluteError = absoluteSum / expected.Length,\n\t\t\tMaximumRelativeError = maxRelative,\n\t\t\tMaximumErrorIndex = maxIndex,\n\t\t\tExpectedAtMaximumError = expected[maxIndex],\n\t\t\tActualAtMaximumError = actual[maxIndex],\n\t\t\tFirstFailingIndex = firstFailingIndex,\n\t\t\tExpectedAtFirstFailure = firstFailingIndex < 0 ? 0 : expected[firstFailingIndex],\n\t\t\tActualAtFirstFailure = firstFailingIndex < 0 ? 0 : actual[firstFailingIndex],\n\t\t\tAbsoluteTolerance = absoluteTolerance,\n\t\t\tRelativeTolerance = relativeTolerance,\n\t\t\tPassed = passed\n\t\t};\n\t}\n\n\tprivate static ulong HashByte( ulong hash, byte value )\n\t{\n\t\treturn (hash ^ value) * FnvPrime;\n\t}\n\n\tprivate static string FormatEdge( ReadOnlySpan<float> values, int start, int count )\n\t{\n\t\tStringBuilder builder = new();\n\t\tbuilder.Append( '[' );\n\t\tfor ( int index = 0; index < count; index++ )\n\t\t{\n\t\t\tif ( index > 0 )\n\t\t\t{\n\t\t\t\tbuilder.Append( ',' );\n\t\t\t}\n\t\t\tbuilder.Append( values[start + index].ToString( \"G9\" ) );\n\t\t}\n\t\tbuilder.Append( ']' );\n\t\treturn builder.ToString();\n\t}\n}\n"
},
{
"Ident": "jeffskitchen.llm_poc",
"Path": "Llm/TinyStoriesForwardStages.cs",
"FileName": "TinyStoriesForwardStages.cs",
"PackageType": "game",
"CodeKind": "Game",
"AssetVersionId": 342521,
"Code": "namespace LlmPoc.Llm;\n\npublic static class TinyStoriesForwardStages\n{\n\tpublic const string TokenEmbeddingName = \"transformer.wte.weight\";\n\tpublic const string PositionEmbeddingName = \"transformer.wpe.weight\";\n\tpublic const string Layer0Ln1WeightName = \"transformer.h.0.ln_1.weight\";\n\tpublic const string Layer0Ln1BiasName = \"transformer.h.0.ln_1.bias\";\n\tpublic const string Layer0QWeightName = \"transformer.h.0.attn.attention.q_proj.weight\";\n\tpublic const string Layer0KWeightName = \"transformer.h.0.attn.attention.k_proj.weight\";\n\tpublic const string Layer0VWeightName = \"transformer.h.0.attn.attention.v_proj.weight\";\n\tpublic const string Layer0AttentionOutProjectionWeightName =\n\t\t\"transformer.h.0.attn.attention.out_proj.weight\";\n\tpublic const string Layer0AttentionOutProjectionBiasName =\n\t\t\"transformer.h.0.attn.attention.out_proj.bias\";\n\tpublic const string Layer0Ln2WeightName = \"transformer.h.0.ln_2.weight\";\n\tpublic const string Layer0Ln2BiasName = \"transformer.h.0.ln_2.bias\";\n\tpublic const string Layer0MlpFcWeightName = \"transformer.h.0.mlp.c_fc.weight\";\n\tpublic const string Layer0MlpFcBiasName = \"transformer.h.0.mlp.c_fc.bias\";\n\tpublic const string Layer0MlpProjWeightName = \"transformer.h.0.mlp.c_proj.weight\";\n\tpublic const string Layer0MlpProjBiasName = \"transformer.h.0.mlp.c_proj.bias\";\n\tpublic const float GeluNewTanhCoefficient = 0.7978845608028654f;\n\tpublic const float GeluNewCubicCoefficient = 0.044715f;\n\tpublic const string GeluNewFormula =\n\t\t\"0.5*x*(1.0+tanh(sqrt(2.0/pi)*(x+0.044715*pow(x,3.0))))\";\n\n\tpublic static Tensor CombineEmbeddings(\n\t\tSboxLlmModel model,\n\t\tTinyStoriesConfig config,\n\t\tIReadOnlyList<int> tokenIds,\n\t\tbool logDiagnostics = true )\n\t{\n\t\tif ( model is null )\n\t\t{\n\t\t\tthrow new ArgumentNullException( nameof( model ) );\n\t\t}\n\t\tif ( config is null )\n\t\t{\n\t\t\tthrow new ArgumentNullException( nameof( config ) );\n\t\t}\n\t\tif ( tokenIds is null || tokenIds.Count == 0 )\n\t\t{\n\t\t\tthrow new ArgumentException( \"[LLM:ERROR] Embedding input token IDs cannot be empty.\" );\n\t\t}\n\t\tif ( tokenIds.Count > config.MaximumPositions )\n\t\t{\n\t\t\tthrow new InvalidOperationException(\n\t\t\t\t$\"[LLM:ERROR] Embedding sequence length {tokenIds.Count} exceeds \" +\n\t\t\t\t$\"maximum positions {config.MaximumPositions}.\" );\n\t\t}\n\n\t\tTensor tokenEmbedding = model.GetRequiredTensor( TokenEmbeddingName );\n\t\tTensor positionEmbedding = model.GetRequiredTensor( PositionEmbeddingName );\n\t\ttokenEmbedding.RequireShape( config.VocabularySize, config.HiddenSize );\n\t\tpositionEmbedding.RequireShape( config.MaximumPositions, config.HiddenSize );\n\n\t\tif ( tokenIds.Count > int.MaxValue / config.HiddenSize )\n\t\t{\n\t\t\tthrow new InvalidOperationException(\n\t\t\t\t$\"[LLM:ERROR] Embedding output shape [{tokenIds.Count},{config.HiddenSize}] \" +\n\t\t\t\t\"exceeds the managed array limit.\" );\n\t\t}\n\n\t\tif ( logDiagnostics )\n\t\t{\n\t\t\tLlmLog.Info(\n\t\t\t\t\"EMBED\",\n\t\t\t\t$\"input shape=[{tokenIds.Count}] tokens=[{string.Join( \",\", tokenIds )}] \" +\n\t\t\t\t$\"positions=[0..{tokenIds.Count - 1}] token_tensor={TokenEmbeddingName}{tokenEmbedding.ShapeText} \" +\n\t\t\t\t$\"position_tensor={PositionEmbeddingName}{positionEmbedding.ShapeText}\" );\n\t\t}\n\n\t\tint hiddenSize = config.HiddenSize;\n\t\tfloat[] output = new float[tokenIds.Count * hiddenSize];\n\t\tfor ( int position = 0; position < tokenIds.Count; position++ )\n\t\t{\n\t\t\tint tokenId = tokenIds[position];\n\t\t\tif ( tokenId < 0 || tokenId >= config.VocabularySize )\n\t\t\t{\n\t\t\t\tthrow new IndexOutOfRangeException(\n\t\t\t\t\t$\"[LLM:ERROR] Embedding token ID {tokenId} at sequence index {position} \" +\n\t\t\t\t\t$\"is outside [0,{config.VocabularySize}).\" );\n\t\t\t}\n\n\t\t\tint tokenRow = tokenId * hiddenSize;\n\t\t\tint positionRow = position * hiddenSize;\n\t\t\tint outputRow = position * hiddenSize;\n\t\t\tfor ( int hidden = 0; hidden < hiddenSize; hidden++ )\n\t\t\t{\n\t\t\t\toutput[outputRow + hidden] =\n\t\t\t\t\ttokenEmbedding.Data[tokenRow + hidden] +\n\t\t\t\t\tpositionEmbedding.Data[positionRow + hidden];\n\t\t\t}\n\t\t}\n\n\t\treturn new Tensor(\n\t\t\t\"forward.combined_embedding\",\n\t\t\tnew[] { tokenIds.Count, hiddenSize },\n\t\t\toutput );\n\t}\n\n\tpublic static Tensor ApplyLayer0Ln1(\n\t\tSboxLlmModel model,\n\t\tTinyStoriesConfig config,\n\t\tTensor input )\n\t{\n\t\treturn ApplyLayerNorm(\n\t\t\tmodel,\n\t\t\tconfig,\n\t\t\tinput,\n\t\t\tLayer0Ln1WeightName,\n\t\t\tLayer0Ln1BiasName,\n\t\t\t\"layer=0 ln=1\",\n\t\t\t\"Layer 0 ln_1\",\n\t\t\t\"forward.layer0.ln_1\" );\n\t}\n\n\tpublic static Tensor ApplyLayer0Ln2(\n\t\tSboxLlmModel model,\n\t\tTinyStoriesConfig config,\n\t\tTensor input )\n\t{\n\t\treturn ApplyLayerNorm(\n\t\t\tmodel,\n\t\t\tconfig,\n\t\t\tinput,\n\t\t\tLayer0Ln2WeightName,\n\t\t\tLayer0Ln2BiasName,\n\t\t\t\"layer=0 ln=2\",\n\t\t\t\"Layer 0 ln_2\",\n\t\t\t\"forward.layer0.ln_2\" );\n\t}\n\n\tpublic static Tensor ApplyLayerNorm(\n\t\tSboxLlmModel model,\n\t\tTinyStoriesConfig config,\n\t\tTensor input,\n\t\tint layerIndex,\n\t\tint normIndex,\n\t\tstring outputName,\n\t\tbool logDiagnostics = true )\n\t{\n\t\tif ( layerIndex < 0 || layerIndex >= config.LayerCount )\n\t\t{\n\t\t\tthrow new IndexOutOfRangeException(\n\t\t\t\t$\"[LLM:ERROR] LayerNorm layer index {layerIndex} is outside \" +\n\t\t\t\t$\"[0,{config.LayerCount}).\" );\n\t\t}\n\t\tif ( normIndex != 1 && normIndex != 2 )\n\t\t{\n\t\t\tthrow new ArgumentOutOfRangeException(\n\t\t\t\tnameof( normIndex ), normIndex,\n\t\t\t\t\"[LLM:ERROR] GPT-Neo transformer LayerNorm index must be 1 or 2.\" );\n\t\t}\n\n\t\tstring prefix = $\"transformer.h.{layerIndex}.ln_{normIndex}\";\n\t\treturn ApplyLayerNorm(\n\t\t\tmodel,\n\t\t\tconfig,\n\t\t\tinput,\n\t\t\t$\"{prefix}.weight\",\n\t\t\t$\"{prefix}.bias\",\n\t\t\t$\"layer={layerIndex} ln={normIndex}\",\n\t\t\t$\"Layer {layerIndex} ln_{normIndex}\",\n\t\t\toutputName,\n\t\t\tlogDiagnostics );\n\t}\n\n\tpublic static Tensor ApplyFinalLayerNorm(\n\t\tSboxLlmModel model,\n\t\tTinyStoriesConfig config,\n\t\tTensor input,\n\t\tstring outputName,\n\t\tbool logDiagnostics = true )\n\t{\n\t\treturn ApplyLayerNorm(\n\t\t\tmodel,\n\t\t\tconfig,\n\t\t\tinput,\n\t\t\t\"transformer.ln_f.weight\",\n\t\t\t\"transformer.ln_f.bias\",\n\t\t\t\"stage=final_ln\",\n\t\t\t\"Final model LayerNorm\",\n\t\t\toutputName,\n\t\t\tlogDiagnostics );\n\t}\n\n\tprivate static Tensor ApplyLayerNorm(\n\t\tSboxLlmModel model,\n\t\tTinyStoriesConfig config,\n\t\tTensor input,\n\t\tstring weightName,\n\t\tstring biasName,\n\t\tstring logLabel,\n\t\tstring errorLabel,\n\t\tstring outputName,\n\t\tbool logDiagnostics = true )\n\t{\n\t\tif ( model is null ) throw new ArgumentNullException( nameof( model ) );\n\t\tif ( config is null ) throw new ArgumentNullException( nameof( config ) );\n\t\tif ( input is null ) throw new ArgumentNullException( nameof( input ) );\n\t\tif ( input.Rank != 2 || input.Shape[0] <= 0 || input.Shape[1] != config.HiddenSize )\n\t\t{\n\t\t\tthrow new InvalidOperationException(\n\t\t\t\t$\"[LLM:ERROR] {errorLabel} input expected shape [sequence,{config.HiddenSize}] \" +\n\t\t\t\t$\"with a positive sequence length, found {input.ShapeText}.\" );\n\t\t}\n\n\t\tTensor weight = model.GetRequiredTensor( weightName );\n\t\tTensor bias = model.GetRequiredTensor( biasName );\n\t\tweight.RequireShape( config.HiddenSize );\n\t\tbias.RequireShape( config.HiddenSize );\n\t\tif ( !(config.LayerNormEpsilon > 0) || !float.IsFinite( config.LayerNormEpsilon ) )\n\t\t{\n\t\t\tthrow new InvalidOperationException(\n\t\t\t\t$\"[LLM:ERROR] {errorLabel} epsilon must be positive and finite, \" +\n\t\t\t\t$\"found {config.LayerNormEpsilon:G9}.\" );\n\t\t}\n\n\t\tint sequenceLength = input.Shape[0];\n\t\tint hiddenSize = config.HiddenSize;\n\t\tfloat[] output = new float[input.Data.Length];\n\t\tif ( logDiagnostics )\n\t\t{\n\t\t\tLlmLog.Info(\n\t\t\t\t\"LN\",\n\t\t\t\t$\"{logLabel} input_shape={input.ShapeText} expected_shape=[{sequenceLength},{hiddenSize}] \" +\n\t\t\t\t$\"weight={weightName}{weight.ShapeText} bias={biasName}{bias.ShapeText} \" +\n\t\t\t\t$\"epsilon={config.LayerNormEpsilon:G9} variance=population(unbiased=false)\" );\n\t\t}\n\n\t\tfor ( int token = 0; token < sequenceLength; token++ )\n\t\t{\n\t\t\tint row = token * hiddenSize;\n\t\t\t// PyTorch 2.9.1's installed AVX2 LayerNorm kernel uses RowwiseMoments:\n\t\t\t// eight FP32 Welford lanes over this model's 64 hidden values, followed\n\t\t\t// by a left-to-right cascade of the lane moments. This scalar spelling\n\t\t\t// reproduces that numerical reduction order without introducing SIMD.\n\t\t\t(float mean, float variance) = ComputeAvx2RowwiseMoments64( input.Data, row );\n\t\t\tfloat denominator = MathF.Sqrt( variance + config.LayerNormEpsilon );\n\t\t\tif ( !(denominator > 0) || !float.IsFinite( denominator ) )\n\t\t\t{\n\t\t\t\tthrow new InvalidOperationException(\n\t\t\t\t\t$\"[LLM:ERROR] {errorLabel} token {token} produced invalid denominator \" +\n\t\t\t\t\t$\"{denominator:G9} from mean={mean:G9}, variance={variance:G9}, \" +\n\t\t\t\t\t$\"epsilon={config.LayerNormEpsilon:G9}.\" );\n\t\t\t}\n\t\t\tfloat inverseStandardDeviation = 1.0f / denominator;\n\n\t\t\tfor ( int hidden = 0; hidden < hiddenSize; hidden++ )\n\t\t\t{\n\t\t\t\tfloat normalized = (input.Data[row + hidden] - mean) * inverseStandardDeviation;\n\t\t\t\toutput[row + hidden] = normalized * weight.Data[hidden] + bias.Data[hidden];\n\t\t\t}\n\n\t\t\tif ( logDiagnostics && token == 0 )\n\t\t\t{\n\t\t\t\tLlmLog.Trace(\n\t\t\t\t\t\"LN\",\n\t\t\t\t\t$\"{logLabel} token=0 mean={mean:G12} variance={variance:G12} \" +\n\t\t\t\t\t$\"denominator={denominator:G12} inverse_std={inverseStandardDeviation:G12} \" +\n\t\t\t\t\t$\"output_first=[{output[row]:G9},{output[row + 1]:G9},\" +\n\t\t\t\t\t$\"{output[row + 2]:G9},{output[row + 3]:G9}]\" );\n\t\t\t}\n\t\t}\n\n\t\treturn new Tensor(\n\t\t\toutputName,\n\t\t\tnew[] { sequenceLength, hiddenSize },\n\t\t\toutput );\n\t}\n\n\tprivate static (float Mean, float Variance) ComputeAvx2RowwiseMoments64(\n\t\tfloat[] values,\n\t\tint rowOffset )\n\t{\n\t\tconst int laneCount = 8;\n\t\tconst int valuesPerLane = 8;\n\t\tfloat[] laneMeans = new float[laneCount];\n\t\tfloat[] laneMoment2 = new float[laneCount];\n\n\t\tfor ( int item = 0; item < valuesPerLane; item++ )\n\t\t{\n\t\t\tfloat reciprocalCount = 1.0f / (item + 1);\n\t\t\tint itemOffset = rowOffset + item * laneCount;\n\t\t\tfor ( int lane = 0; lane < laneCount; lane++ )\n\t\t\t{\n\t\t\t\tfloat value = values[itemOffset + lane];\n\t\t\t\tfloat delta = value - laneMeans[lane];\n\t\t\t\tfloat meanIncrement = delta * reciprocalCount;\n\t\t\t\tlaneMeans[lane] += meanIncrement;\n\t\t\t\tfloat remainingDelta = value - laneMeans[lane];\n\t\t\t\tfloat momentIncrement = delta * remainingDelta;\n\t\t\t\tlaneMoment2[lane] += momentIncrement;\n\t\t\t}\n\t\t}\n\n\t\tint accumulatedCount = 0;\n\t\tfloat mean = 0.0f;\n\t\tfloat moment2 = 0.0f;\n\t\tfor ( int lane = 0; lane < laneCount; lane++ )\n\t\t{\n\t\t\tint combinedCount = accumulatedCount + valuesPerLane;\n\t\t\tfloat contribution = (float)valuesPerLane / combinedCount;\n\t\t\tfloat delta = laneMeans[lane] - mean;\n\t\t\tfloat meanIncrement = contribution * delta;\n\t\t\tmean += meanIncrement;\n\n\t\t\tfloat deltaSquared = delta * delta;\n\t\t\tfloat weightedDelta = deltaSquared * contribution;\n\t\t\tweightedDelta *= accumulatedCount;\n\t\t\tfloat combinedMoment = laneMoment2[lane] + weightedDelta;\n\t\t\tmoment2 += combinedMoment;\n\t\t\taccumulatedCount = combinedCount;\n\t\t}\n\n\t\treturn (mean, moment2 / 64.0f);\n\t}\n\n\tpublic static Tensor LinearNoBias( Tensor input, Tensor weight, string outputName )\n\t{\n\t\treturn Linear( input, weight, null, outputName );\n\t}\n\n\tpublic static Tensor LinearWithBias(\n\t\tTensor input,\n\t\tTensor weight,\n\t\tTensor bias,\n\t\tstring outputName )\n\t{\n\t\tif ( bias is null )\n\t\t{\n\t\t\tthrow new ArgumentNullException( nameof( bias ) );\n\t\t}\n\t\treturn Linear( input, weight, bias, outputName );\n\t}\n\n\tprivate static Tensor Linear(\n\t\tTensor input,\n\t\tTensor weight,\n\t\tTensor bias,\n\t\tstring outputName )\n\t{\n\t\tif ( input is null )\n\t\t{\n\t\t\tthrow new ArgumentNullException( nameof( input ) );\n\t\t}\n\t\tif ( weight is null )\n\t\t{\n\t\t\tthrow new ArgumentNullException( nameof( weight ) );\n\t\t}\n\t\tif ( string.IsNullOrWhiteSpace( outputName ) )\n\t\t{\n\t\t\tthrow new ArgumentException(\n\t\t\t\t\"[LLM:ERROR] Linear projection output name cannot be empty.\",\n\t\t\t\tnameof( outputName ) );\n\t\t}\n\t\tif ( input.Rank != 2 )\n\t\t{\n\t\t\tthrow new InvalidOperationException(\n\t\t\t\t$\"[LLM:ERROR] {outputName} linear input expected rank 2 \" +\n\t\t\t\t$\"[sequence,input_size], found {input.ShapeText}.\" );\n\t\t}\n\t\tif ( weight.Rank != 2 )\n\t\t{\n\t\t\tthrow new InvalidOperationException(\n\t\t\t\t$\"[LLM:ERROR] {outputName} weight '{weight.Name}' expected rank 2 \" +\n\t\t\t\t$\"[output_size,input_size], found {weight.ShapeText}.\" );\n\t\t}\n\n\t\tint sequenceLength = input.Shape[0];\n\t\tint inputSize = input.Shape[1];\n\t\tint outputSize = weight.Shape[0];\n\t\tif ( weight.Shape[1] != inputSize )\n\t\t{\n\t\t\tthrow new InvalidOperationException(\n\t\t\t\t$\"[LLM:ERROR] {outputName} incompatible linear shapes: input={input.ShapeText}, \" +\n\t\t\t\t$\"weight={weight.ShapeText}; weight input axis expected {inputSize}, \" +\n\t\t\t\t$\"found {weight.Shape[1]}.\" );\n\t\t}\n\t\tif ( bias is not null )\n\t\t{\n\t\t\tif ( bias.Rank != 1 || bias.Shape[0] != outputSize )\n\t\t\t{\n\t\t\t\tthrow new InvalidOperationException(\n\t\t\t\t\t$\"[LLM:ERROR] {outputName} linear bias '{bias.Name}' expected shape \" +\n\t\t\t\t\t$\"[{outputSize}], found {bias.ShapeText}.\" );\n\t\t\t}\n\t\t}\n\t\tif ( sequenceLength > int.MaxValue / outputSize )\n\t\t{\n\t\t\tthrow new InvalidOperationException(\n\t\t\t\t$\"[LLM:ERROR] {outputName} output shape [{sequenceLength},{outputSize}] \" +\n\t\t\t\t\"exceeds the managed array limit.\" );\n\t\t}\n\n\t\tfloat[] output = new float[sequenceLength * outputSize];\n\t\tfor ( int token = 0; token < sequenceLength; token++ )\n\t\t{\n\t\t\tint inputRow = token * inputSize;\n\t\t\tint outputRow = token * outputSize;\n\t\t\tfor ( int outputFeature = 0; outputFeature < outputSize; outputFeature++ )\n\t\t\t{\n\t\t\t\tint weightRow = outputFeature * inputSize;\n\t\t\t\t// The installed PyTorch 2.9.1 CPU Linear path was independently\n\t\t\t\t// checked against the live module outputs. For this model it is\n\t\t\t\t// bit-identical to a left-to-right FP32 fused multiply-add reduction.\n\t\t\t\t// The small attention matmuls intentionally retain their separately\n\t\t\t\t// validated non-fused multiply/add loops.\n\t\t\t\tfloat sum = 0.0f;\n\t\t\t\tfor ( int inputFeature = 0; inputFeature < inputSize; inputFeature++ )\n\t\t\t\t{\n\t\t\t\t\tsum = MathF.FusedMultiplyAdd(\n\t\t\t\t\t\tinput.Data[inputRow + inputFeature],\n\t\t\t\t\t\tweight.Data[weightRow + inputFeature],\n\t\t\t\t\t\tsum );\n\t\t\t\t}\n\t\t\t\toutput[outputRow + outputFeature] =\n\t\t\t\t\tbias is null ? sum : sum + bias.Data[outputFeature];\n\t\t\t}\n\t\t}\n\n\t\treturn new Tensor(\n\t\t\toutputName,\n\t\t\tnew[] { sequenceLength, outputSize },\n\t\t\toutput );\n\t}\n\n\tpublic static string DescribeLinearDotProduct(\n\t\tTensor input,\n\t\tTensor weight,\n\t\tint token,\n\t\tint outputFeature,\n\t\tfloat expected,\n\t\tfloat actual,\n\t\tTensor bias = null )\n\t{\n\t\tif ( input is null || weight is null || input.Rank != 2 || weight.Rank != 2 ||\n\t\t\tweight.Shape[1] != input.Shape[1] )\n\t\t{\n\t\t\tthrow new InvalidOperationException(\n\t\t\t\t\"[LLM:ERROR] Dot-product diagnostic requires compatible rank-2 input and weight tensors.\" );\n\t\t}\n\t\tif ( bias is not null && (bias.Rank != 1 || bias.Shape[0] != weight.Shape[0]) )\n\t\t{\n\t\t\tthrow new InvalidOperationException(\n\t\t\t\t$\"[LLM:ERROR] Dot-product diagnostic bias '{bias.Name}' expected shape \" +\n\t\t\t\t$\"[{weight.Shape[0]}], found {bias.ShapeText}.\" );\n\t\t}\n\t\tif ( token < 0 || token >= input.Shape[0] ||\n\t\t\toutputFeature < 0 || outputFeature >= weight.Shape[0] )\n\t\t{\n\t\t\tthrow new IndexOutOfRangeException(\n\t\t\t\t$\"[LLM:ERROR] Dot-product diagnostic index token={token}, \" +\n\t\t\t\t$\"output_feature={outputFeature} is outside input={input.ShapeText}, \" +\n\t\t\t\t$\"weight={weight.ShapeText}.\" );\n\t\t}\n\n\t\tint inputSize = input.Shape[1];\n\t\tint inputRow = token * inputSize;\n\t\tint weightRow = outputFeature * inputSize;\n\t\tfloat sum = 0;\n\t\tfloat product0 = input.Data[inputRow] * weight.Data[weightRow];\n\t\tfloat product1 = input.Data[inputRow + 1] * weight.Data[weightRow + 1];\n\t\tfloat product2 = input.Data[inputRow + 2] * weight.Data[weightRow + 2];\n\t\tfloat product3 = input.Data[inputRow + 3] * weight.Data[weightRow + 3];\n\t\tfor ( int inputFeature = 0; inputFeature < inputSize; inputFeature++ )\n\t\t{\n\t\t\tsum = MathF.FusedMultiplyAdd(\n\t\t\t\tinput.Data[inputRow + inputFeature],\n\t\t\t\tweight.Data[weightRow + inputFeature],\n\t\t\t\tsum );\n\t\t}\n\n\t\tfloat biasValue = bias is null ? 0 : bias.Data[outputFeature];\n\t\tfloat recomputed = bias is null ? sum : sum + biasValue;\n\t\tstring formula = bias is null\n\t\t\t? \"sum(input[token,i]*weight[out,i])\"\n\t\t\t: \"sum(input[token,i]*weight[out,i])+bias[out]\";\n\t\treturn $\"token={token} output_feature={outputFeature} input_length={inputSize} \" +\n\t\t\t$\"formula={formula} expected={expected:G9} actual={actual:G9} \" +\n\t\t\t$\"dot={sum:G9} bias={biasValue:G9} recomputed={recomputed:G9} \" +\n\t\t\t$\"first_products=[{product0:G9},{product1:G9},{product2:G9},{product3:G9}]\";\n\t}\n\n\tpublic static Tensor ApplyGeluNew( Tensor input, string outputName )\n\t{\n\t\tif ( input is null )\n\t\t{\n\t\t\tthrow new ArgumentNullException( nameof( input ) );\n\t\t}\n\t\tif ( string.IsNullOrWhiteSpace( outputName ) )\n\t\t{\n\t\t\tthrow new ArgumentException(\n\t\t\t\t\"[LLM:ERROR] GELU-new output name cannot be empty.\",\n\t\t\t\tnameof( outputName ) );\n\t\t}\n\t\tif ( input.Rank != 2 || input.Shape[0] <= 0 || input.Shape[1] <= 0 )\n\t\t{\n\t\t\tthrow new InvalidOperationException(\n\t\t\t\t$\"[LLM:ERROR] {outputName} GELU-new input expected a positive rank-2 \" +\n\t\t\t\t$\"[sequence,feature] tensor, found {input.ShapeText}.\" );\n\t\t}\n\n\t\tfloat[] output = new float[input.Data.Length];\n\t\tfor ( int index = 0; index < input.Data.Length; index++ )\n\t\t{\n\t\t\tfloat value = input.Data[index];\n\t\t\tif ( !float.IsFinite( value ) )\n\t\t\t{\n\t\t\t\tthrow new InvalidOperationException(\n\t\t\t\t\t$\"[LLM:ERROR] {outputName} GELU-new input contains non-finite \" +\n\t\t\t\t\t$\"value at flat index {index}: {value}.\" );\n\t\t\t}\n\t\t\tfloat cube = MathF.Pow( value, 3.0f );\n\t\t\tfloat inner = value + GeluNewCubicCoefficient * cube;\n\t\t\tfloat tanhArgument = GeluNewTanhCoefficient * inner;\n\t\t\tfloat tanhValue = MathF.Tanh( tanhArgument );\n\t\t\tfloat activated = (0.5f * value) * (1.0f + tanhValue);\n\t\t\tif ( !float.IsFinite( activated ) )\n\t\t\t{\n\t\t\t\tthrow new InvalidOperationException(\n\t\t\t\t\t$\"[LLM:ERROR] {outputName} GELU-new produced non-finite output \" +\n\t\t\t\t\t$\"at flat index {index}: input={value:G9}, cube={cube:G9}, \" +\n\t\t\t\t\t$\"tanh_argument={tanhArgument:G9}, output={activated}.\" );\n\t\t\t}\n\t\t\toutput[index] = activated;\n\t\t}\n\n\t\treturn new Tensor(\n\t\t\toutputName,\n\t\t\tnew[] { input.Shape[0], input.Shape[1] },\n\t\t\toutput );\n\t}\n\n\tpublic static string DescribeGeluNewElement(\n\t\tTensor input,\n\t\tTensor output,\n\t\tint token,\n\t\tint feature,\n\t\tfloat expected )\n\t{\n\t\tif ( input is null || output is null || input.Rank != 2 || output.Rank != 2 ||\n\t\t\tinput.Shape[0] != output.Shape[0] || input.Shape[1] != output.Shape[1] )\n\t\t{\n\t\t\tthrow new InvalidOperationException(\n\t\t\t\t\"[LLM:ERROR] GELU-new diagnostic requires matching rank-2 tensors.\" );\n\t\t}\n\t\tif ( token < 0 || token >= input.Shape[0] || feature < 0 || feature >= input.Shape[1] )\n\t\t{\n\t\t\tthrow new IndexOutOfRangeException(\n\t\t\t\t$\"[LLM:ERROR] GELU-new diagnostic index [{token},{feature}] is \" +\n\t\t\t\t$\"outside {input.ShapeText}.\" );\n\t\t}\n\n\t\tint index = token * input.Shape[1] + feature;\n\t\tfloat value = input.Data[index];\n\t\tfloat cube = MathF.Pow( value, 3.0f );\n\t\tfloat inner = value + GeluNewCubicCoefficient * cube;\n\t\tfloat tanhArgument = GeluNewTanhCoefficient * inner;\n\t\tfloat tanhValue = MathF.Tanh( tanhArgument );\n\t\tfloat recomputed = (0.5f * value) * (1.0f + tanhValue);\n\t\tfloat actual = output.Data[index];\n\t\treturn $\"token={token} feature={feature} input={value:G9} cube={cube:G9} \" +\n\t\t\t$\"cubic_coefficient={GeluNewCubicCoefficient:G9} inner={inner:G9} \" +\n\t\t\t$\"tanh_coefficient={GeluNewTanhCoefficient:G9} \" +\n\t\t\t$\"tanh_argument={tanhArgument:G9} tanh={tanhValue:G9} \" +\n\t\t\t$\"recomputed={recomputed:G9} actual={actual:G9} python={expected:G9} \" +\n\t\t\t$\"abs_diff={MathF.Abs( actual - expected ):G12}\";\n\t}\n\n\tpublic static Tensor AddMlpResidual(\n\t\tTensor residualSource,\n\t\tTensor mlpBranch,\n\t\tstring outputName )\n\t{\n\t\tif ( residualSource is null )\n\t\t{\n\t\t\tthrow new ArgumentNullException( nameof( residualSource ) );\n\t\t}\n\t\tif ( mlpBranch is null )\n\t\t{\n\t\t\tthrow new ArgumentNullException( nameof( mlpBranch ) );\n\t\t}\n\t\tif ( string.IsNullOrWhiteSpace( outputName ) )\n\t\t{\n\t\t\tthrow new ArgumentException(\n\t\t\t\t\"[LLM:ERROR] MLP residual output name cannot be empty.\",\n\t\t\t\tnameof( outputName ) );\n\t\t}\n\t\tif ( residualSource.Rank != 2 || mlpBranch.Rank != 2 ||\n\t\t\tresidualSource.Shape[0] != mlpBranch.Shape[0] ||\n\t\t\tresidualSource.Shape[1] != mlpBranch.Shape[1] ||\n\t\t\tresidualSource.Data.Length != mlpBranch.Data.Length )\n\t\t{\n\t\t\tthrow new InvalidOperationException(\n\t\t\t\t$\"[LLM:ERROR] {outputName} MLP residual addition requires matching \" +\n\t\t\t\t$\"rank-2 tensors, found residual={residualSource.ShapeText}, \" +\n\t\t\t\t$\"MLP={mlpBranch.ShapeText}.\" );\n\t\t}\n\n\t\tfloat[] output = new float[residualSource.Data.Length];\n\t\tfor ( int index = 0; index < output.Length; index++ )\n\t\t{\n\t\t\t// Match GPTNeoBlock.forward: residual + feed_forward_hidden_states.\n\t\t\toutput[index] = residualSource.Data[index] + mlpBranch.Data[index];\n\t\t}\n\t\treturn new Tensor(\n\t\t\toutputName,\n\t\t\tnew[] { residualSource.Shape[0], residualSource.Shape[1] },\n\t\t\toutput );\n\t}\n\n\tpublic static string DescribeMlpResidualAddition(\n\t\tTensor residualSource,\n\t\tTensor mlpBranch,\n\t\tTensor output,\n\t\tint token,\n\t\tint feature,\n\t\tfloat expected )\n\t{\n\t\tif ( residualSource is null || mlpBranch is null || output is null ||\n\t\t\tresidualSource.Rank != 2 || mlpBranch.Rank != 2 || output.Rank != 2 ||\n\t\t\tresidualSource.Shape[0] != mlpBranch.Shape[0] ||\n\t\t\tresidualSource.Shape[1] != mlpBranch.Shape[1] ||\n\t\t\tresidualSource.Shape[0] != output.Shape[0] ||\n\t\t\tresidualSource.Shape[1] != output.Shape[1] )\n\t\t{\n\t\t\tthrow new InvalidOperationException(\n\t\t\t\t\"[LLM:ERROR] MLP residual diagnostic requires matching rank-2 tensors.\" );\n\t\t}\n\t\tif ( token < 0 || token >= output.Shape[0] || feature < 0 || feature >= output.Shape[1] )\n\t\t{\n\t\t\tthrow new IndexOutOfRangeException(\n\t\t\t\t$\"[LLM:ERROR] MLP residual diagnostic index [{token},{feature}] is \" +\n\t\t\t\t$\"outside {output.ShapeText}.\" );\n\t\t}\n\n\t\tint index = token * output.Shape[1] + feature;\n\t\tfloat residual = residualSource.Data[index];\n\t\tfloat branch = mlpBranch.Data[index];\n\t\tfloat actual = output.Data[index];\n\t\treturn $\"token={token} feature={feature} residual_source={residual:G9} \" +\n\t\t\t$\"mlp_branch={branch:G9} sum={actual:G9} python={expected:G9} \" +\n\t\t\t$\"abs_diff={MathF.Abs( expected - actual ):G12}\";\n\t}\n\n\tpublic static Tensor AddResidual(\n\t\tTensor attentionBranch,\n\t\tTensor residualSource,\n\t\tstring outputName )\n\t{\n\t\tif ( attentionBranch is null )\n\t\t{\n\t\t\tthrow new ArgumentNullException( nameof( attentionBranch ) );\n\t\t}\n\t\tif ( residualSource is null )\n\t\t{\n\t\t\tthrow new ArgumentNullException( nameof( residualSource ) );\n\t\t}\n\t\tif ( string.IsNullOrWhiteSpace( outputName ) )\n\t\t{\n\t\t\tthrow new ArgumentException(\n\t\t\t\t\"[LLM:ERROR] Residual-add output name cannot be empty.\",\n\t\t\t\tnameof( outputName ) );\n\t\t}\n\t\tif ( attentionBranch.Rank != 2 || residualSource.Rank != 2 ||\n\t\t\tattentionBranch.Shape[0] != residualSource.Shape[0] ||\n\t\t\tattentionBranch.Shape[1] != residualSource.Shape[1] ||\n\t\t\tattentionBranch.Data.Length != residualSource.Data.Length )\n\t\t{\n\t\t\tthrow new InvalidOperationException(\n\t\t\t\t$\"[LLM:ERROR] {outputName} residual addition requires matching rank-2 \" +\n\t\t\t\t$\"tensors, found attention={attentionBranch.ShapeText}, \" +\n\t\t\t\t$\"residual={residualSource.ShapeText}.\" );\n\t\t}\n\n\t\tfloat[] output = new float[attentionBranch.Data.Length];\n\t\tfor ( int index = 0; index < output.Length; index++ )\n\t\t{\n\t\t\toutput[index] = attentionBranch.Data[index] + residualSource.Data[index];\n\t\t}\n\t\treturn new Tensor(\n\t\t\toutputName,\n\t\t\tnew[] { attentionBranch.Shape[0], attentionBranch.Shape[1] },\n\t\t\toutput );\n\t}\n\n\tpublic static string DescribeResidualAddition(\n\t\tTensor attentionBranch,\n\t\tTensor residualSource,\n\t\tTensor output,\n\t\tint token,\n\t\tint feature,\n\t\tfloat expected )\n\t{\n\t\tif ( attentionBranch is null || residualSource is null || output is null ||\n\t\t\tattentionBranch.Rank != 2 || residualSource.Rank != 2 || output.Rank != 2 ||\n\t\t\tattentionBranch.Shape[0] != residualSource.Shape[0] ||\n\t\t\tattentionBranch.Shape[1] != residualSource.Shape[1] ||\n\t\t\tattentionBranch.Shape[0] != output.Shape[0] ||\n\t\t\tattentionBranch.Shape[1] != output.Shape[1] )\n\t\t{\n\t\t\tthrow new InvalidOperationException(\n\t\t\t\t\"[LLM:ERROR] Residual diagnostic requires compatible rank-2 attention, \" +\n\t\t\t\t\"residual, and output tensors.\" );\n\t\t}\n\t\tif ( token < 0 || token >= output.Shape[0] ||\n\t\t\tfeature < 0 || feature >= output.Shape[1] )\n\t\t{\n\t\t\tthrow new IndexOutOfRangeException(\n\t\t\t\t$\"[LLM:ERROR] Residual diagnostic index [{token},{feature}] is \" +\n\t\t\t\t$\"outside output shape {output.ShapeText}.\" );\n\t\t}\n\n\t\tint index = token * output.Shape[1] + feature;\n\t\tfloat branch = attentionBranch.Data[index];\n\t\tfloat residual = residualSource.Data[index];\n\t\tfloat actual = output.Data[index];\n\t\treturn $\"token={token} feature={feature} residual_source={residual:G9} \" +\n\t\t\t$\"attention_branch={branch:G9} sum={actual:G9} python={expected:G9} \" +\n\t\t\t$\"abs_diff={MathF.Abs( expected - actual ):G12}\";\n\t}\n\n\tpublic static Tensor SplitHeads( Tensor projection, int headCount, string outputName )\n\t{\n\t\tif ( projection is null )\n\t\t{\n\t\t\tthrow new ArgumentNullException( nameof( projection ) );\n\t\t}\n\t\tif ( string.IsNullOrWhiteSpace( outputName ) )\n\t\t{\n\t\t\tthrow new ArgumentException(\n\t\t\t\t\"[LLM:ERROR] Head-split output name cannot be empty.\",\n\t\t\t\tnameof( outputName ) );\n\t\t}\n\t\tif ( projection.Rank != 2 || projection.Shape[0] <= 0 || projection.Shape[1] <= 0 )\n\t\t{\n\t\t\tthrow new InvalidOperationException(\n\t\t\t\t$\"[LLM:ERROR] {outputName} head split expected projection shape \" +\n\t\t\t\t$\"[sequence,hidden] with positive dimensions, found {projection.ShapeText}.\" );\n\t\t}\n\t\tif ( headCount <= 0 || projection.Shape[1] % headCount != 0 )\n\t\t{\n\t\t\tthrow new InvalidOperationException(\n\t\t\t\t$\"[LLM:ERROR] {outputName} hidden size {projection.Shape[1]} must be divisible \" +\n\t\t\t\t$\"by positive head count {headCount}.\" );\n\t\t}\n\n\t\tint sequenceLength = projection.Shape[0];\n\t\tint hiddenSize = projection.Shape[1];\n\t\tint headDimension = hiddenSize / headCount;\n\t\tfloat[] output = new float[projection.Data.Length];\n\t\tfor ( int head = 0; head < headCount; head++ )\n\t\t{\n\t\t\tfor ( int token = 0; token < sequenceLength; token++ )\n\t\t\t{\n\t\t\t\tint sourceRow = token * hiddenSize;\n\t\t\t\tint destinationRow = (head * sequenceLength + token) * headDimension;\n\t\t\t\tfor ( int component = 0; component < headDimension; component++ )\n\t\t\t\t{\n\t\t\t\t\tint sourceFeature = head * headDimension + component;\n\t\t\t\t\toutput[destinationRow + component] = projection.Data[sourceRow + sourceFeature];\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\n\t\treturn new Tensor(\n\t\t\toutputName,\n\t\t\tnew[] { headCount, sequenceLength, headDimension },\n\t\t\toutput );\n\t}\n\n\tpublic static string DescribeHeadMapping(\n\t\tTensor projection,\n\t\tTensor heads,\n\t\tint head,\n\t\tint token,\n\t\tint component )\n\t{\n\t\tif ( projection is null || heads is null || projection.Rank != 2 || heads.Rank != 3 )\n\t\t{\n\t\t\tthrow new InvalidOperationException(\n\t\t\t\t\"[LLM:ERROR] Head mapping diagnostic requires rank-2 projection and rank-3 heads.\" );\n\t\t}\n\t\tint headCount = heads.Shape[0];\n\t\tint sequenceLength = heads.Shape[1];\n\t\tint headDimension = heads.Shape[2];\n\t\tif ( projection.Shape[0] != sequenceLength ||\n\t\t\tprojection.Shape[1] != headCount * headDimension )\n\t\t{\n\t\t\tthrow new InvalidOperationException(\n\t\t\t\t$\"[LLM:ERROR] Head mapping diagnostic incompatible shapes: \" +\n\t\t\t\t$\"projection={projection.ShapeText}, heads={heads.ShapeText}.\" );\n\t\t}\n\t\tif ( head < 0 || head >= headCount || token < 0 || token >= sequenceLength ||\n\t\t\tcomponent < 0 || component >= headDimension )\n\t\t{\n\t\t\tthrow new IndexOutOfRangeException(\n\t\t\t\t$\"[LLM:ERROR] Head mapping diagnostic index [{head},{token},{component}] \" +\n\t\t\t\t$\"is outside {heads.ShapeText}.\" );\n\t\t}\n\n\t\tint sourceFeature = head * headDimension + component;\n\t\tint sourceIndex = token * projection.Shape[1] + sourceFeature;\n\t\tint destinationIndex = (head * sequenceLength + token) * headDimension + component;\n\t\tfloat source = projection.Data[sourceIndex];\n\t\tfloat destination = heads.Data[destinationIndex];\n\t\treturn $\"head={head} token={token} component={component} source_feature={sourceFeature} \" +\n\t\t\t$\"source_index={sourceIndex} destination_index={destinationIndex} \" +\n\t\t\t$\"source={source:G9} destination={destination:G9} \" +\n\t\t\t$\"source_bits={BitConverter.SingleToInt32Bits( source ):X8} \" +\n\t\t\t$\"destination_bits={BitConverter.SingleToInt32Bits( destination ):X8}\";\n\t}\n\n\tpublic static Tensor MergeHeads( Tensor contextHeads, string outputName )\n\t{\n\t\tif ( contextHeads is null )\n\t\t{\n\t\t\tthrow new ArgumentNullException( nameof( contextHeads ) );\n\t\t}\n\t\tif ( string.IsNullOrWhiteSpace( outputName ) )\n\t\t{\n\t\t\tthrow new ArgumentException(\n\t\t\t\t\"[LLM:ERROR] Head-merge output name cannot be empty.\",\n\t\t\t\tnameof( outputName ) );\n\t\t}\n\t\tif ( contextHeads.Rank != 3 || contextHeads.Shape[0] <= 0 ||\n\t\t\tcontextHeads.Shape[1] <= 0 || contextHeads.Shape[2] <= 0 )\n\t\t{\n\t\t\tthrow new InvalidOperationException(\n\t\t\t\t$\"[LLM:ERROR] {outputName} head merge expected positive shape \" +\n\t\t\t\t$\"[head,sequence,head_dimension], found {contextHeads.ShapeText}.\" );\n\t\t}\n\n\t\tint headCount = contextHeads.Shape[0];\n\t\tint sequenceLength = contextHeads.Shape[1];\n\t\tint headDimension = contextHeads.Shape[2];\n\t\tif ( headCount > int.MaxValue / headDimension )\n\t\t{\n\t\t\tthrow new InvalidOperationException(\n\t\t\t\t$\"[LLM:ERROR] {outputName} hidden size {headCount}*{headDimension} \" +\n\t\t\t\t\"exceeds the managed array limit.\" );\n\t\t}\n\t\tint hiddenSize = headCount * headDimension;\n\t\tif ( sequenceLength > int.MaxValue / hiddenSize )\n\t\t{\n\t\t\tthrow new InvalidOperationException(\n\t\t\t\t$\"[LLM:ERROR] {outputName} output shape [{sequenceLength},{hiddenSize}] \" +\n\t\t\t\t\"exceeds the managed array limit.\" );\n\t\t}\n\n\t\tfloat[] output = new float[sequenceLength * hiddenSize];\n\t\tfor ( int token = 0; token < sequenceLength; token++ )\n\t\t{\n\t\t\tint destinationRow = token * hiddenSize;\n\t\t\tfor ( int head = 0; head < headCount; head++ )\n\t\t\t{\n\t\t\t\tint sourceRow = (head * sequenceLength + token) * headDimension;\n\t\t\t\tint destinationHead = destinationRow + head * headDimension;\n\t\t\t\tfor ( int component = 0; component < headDimension; component++ )\n\t\t\t\t{\n\t\t\t\t\toutput[destinationHead + component] =\n\t\t\t\t\t\tcontextHeads.Data[sourceRow + component];\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\n\t\treturn new Tensor(\n\t\t\toutputName,\n\t\t\tnew[] { sequenceLength, hiddenSize },\n\t\t\toutput );\n\t}\n\n\tpublic static string DescribeHeadMergeMapping(\n\t\tTensor contextHeads,\n\t\tTensor merged,\n\t\tint token,\n\t\tint feature )\n\t{\n\t\tif ( contextHeads is null || merged is null ||\n\t\t\tcontextHeads.Rank != 3 || merged.Rank != 2 )\n\t\t{\n\t\t\tthrow new InvalidOperationException(\n\t\t\t\t\"[LLM:ERROR] Head-merge diagnostic requires rank-3 context heads and \" +\n\t\t\t\t\"rank-2 merged context.\" );\n\t\t}\n\t\tint headCount = contextHeads.Shape[0];\n\t\tint sequenceLength = contextHeads.Shape[1];\n\t\tint headDimension = contextHeads.Shape[2];\n\t\tint hiddenSize = headCount * headDimension;\n\t\tif ( merged.Shape[0] != sequenceLength || merged.Shape[1] != hiddenSize )\n\t\t{\n\t\t\tthrow new InvalidOperationException(\n\t\t\t\t$\"[LLM:ERROR] Head-merge diagnostic incompatible shapes: \" +\n\t\t\t\t$\"context={contextHeads.ShapeText}, merged={merged.ShapeText}.\" );\n\t\t}\n\t\tif ( token < 0 || token >= sequenceLength || feature < 0 || feature >= hiddenSize )\n\t\t{\n\t\t\tthrow new IndexOutOfRangeException(\n\t\t\t\t$\"[LLM:ERROR] Head-merge diagnostic index token={token}, feature={feature} \" +\n\t\t\t\t$\"is outside merged shape {merged.ShapeText}.\" );\n\t\t}\n\n\t\tint head = feature / headDimension;\n\t\tint component = feature % headDimension;\n\t\tint sourceIndex = (head * sequenceLength + token) * headDimension + component;\n\t\tint destinationIndex = token * hiddenSize + feature;\n\t\tfloat source = contextHeads.Data[sourceIndex];\n\t\tfloat destination = merged.Data[destinationIndex];\n\t\treturn $\"token={token} feature={feature} head={head} component={component} \" +\n\t\t\t$\"source_index={sourceIndex} destination_index={destinationIndex} \" +\n\t\t\t$\"source={source:G9} merged={destination:G9} \" +\n\t\t\t$\"source_bits={BitConverter.SingleToInt32Bits( source ):X8} \" +\n\t\t\t$\"merged_bits={BitConverter.SingleToInt32Bits( destination ):X8}\";\n\t}\n\n\tpublic static Tensor ComputeScaledUnmaskedAttentionScores(\n\t\tTensor queryHeads,\n\t\tTensor keyHeads,\n\t\tfloat scale,\n\t\tstring outputName )\n\t{\n\t\tif ( queryHeads is null )\n\t\t{\n\t\t\tthrow new ArgumentNullException( nameof( queryHeads ) );\n\t\t}\n\t\tif ( keyHeads is null )\n\t\t{\n\t\t\tthrow new ArgumentNullException( nameof( keyHeads ) );\n\t\t}\n\t\tif ( string.IsNullOrWhiteSpace( outputName ) )\n\t\t{\n\t\t\tthrow new ArgumentException(\n\t\t\t\t\"[LLM:ERROR] Attention-score output name cannot be empty.\",\n\t\t\t\tnameof( outputName ) );\n\t\t}\n\t\tif ( queryHeads.Rank != 3 || keyHeads.Rank != 3 )\n\t\t{\n\t\t\tthrow new InvalidOperationException(\n\t\t\t\t$\"[LLM:ERROR] {outputName} expected Q/K heads rank 3 \" +\n\t\t\t\t$\"[head,sequence,component], found Q={queryHeads.ShapeText}, \" +\n\t\t\t\t$\"K={keyHeads.ShapeText}.\" );\n\t\t}\n\t\tif ( queryHeads.Shape[0] != keyHeads.Shape[0] ||\n\t\t\tqueryHeads.Shape[2] != keyHeads.Shape[2] )\n\t\t{\n\t\t\tthrow new InvalidOperationException(\n\t\t\t\t$\"[LLM:ERROR] {outputName} Q/K head count and component dimensions must match; \" +\n\t\t\t\t$\"found Q={queryHeads.ShapeText}, K={keyHeads.ShapeText}.\" );\n\t\t}\n\t\tif ( !float.IsFinite( scale ) || !(scale > 0) )\n\t\t{\n\t\t\tthrow new InvalidOperationException(\n\t\t\t\t$\"[LLM:ERROR] {outputName} attention scale must be positive and finite, \" +\n\t\t\t\t$\"found {scale:G9}.\" );\n\t\t}\n\n\t\tint headCount = queryHeads.Shape[0];\n\t\tint queryLength = queryHeads.Shape[1];\n\t\tint keyLength = keyHeads.Shape[1];\n\t\tint headDimension = queryHeads.Shape[2];\n\t\tif ( headCount > int.MaxValue / queryLength ||\n\t\t\theadCount * queryLength > int.MaxValue / keyLength )\n\t\t{\n\t\t\tthrow new InvalidOperationException(\n\t\t\t\t$\"[LLM:ERROR] {outputName} shape [{headCount},{queryLength},{keyLength}] \" +\n\t\t\t\t\"exceeds the managed array limit.\" );\n\t\t}\n\n\t\tfloat[] output = new float[headCount * queryLength * keyLength];\n\t\tfor ( int head = 0; head < headCount; head++ )\n\t\t{\n\t\t\tfor ( int query = 0; query < queryLength; query++ )\n\t\t\t{\n\t\t\t\tint queryRow = (head * queryLength + query) * headDimension;\n\t\t\t\tfor ( int key = 0; key < keyLength; key++ )\n\t\t\t\t{\n\t\t\t\t\tint keyRow = (head * keyLength + key) * headDimension;\n\t\t\t\t\tfloat rawSum = 0;\n\t\t\t\t\tfor ( int component = 0; component < headDimension; component++ )\n\t\t\t\t\t{\n\t\t\t\t\t\trawSum += queryHeads.Data[queryRow + component] *\n\t\t\t\t\t\t\tkeyHeads.Data[keyRow + component];\n\t\t\t\t\t}\n\n\t\t\t\t\t// Transformers 5.15 GPT-Neo performs no inverse-sqrt scaling.\n\t\t\t\t\t// Avoid adding an operation when the exact model-equivalent factor is 1.\n\t\t\t\t\toutput[(head * queryLength + query) * keyLength + key] =\n\t\t\t\t\t\tscale == 1.0f ? rawSum : rawSum * scale;\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\n\t\treturn new Tensor(\n\t\t\toutputName,\n\t\t\tnew[] { headCount, queryLength, keyLength },\n\t\t\toutput );\n\t}\n\n\tpublic static bool IsLayer0AttentionAllowed(\n\t\tstring attentionType,\n\t\tint query,\n\t\tint key,\n\t\tint queryLength,\n\t\tint keyLength )\n\t{\n\t\treturn IsAttentionAllowed(\n\t\t\tattentionType,\n\t\t\twindowSize: 0,\n\t\t\tquery,\n\t\t\tkey,\n\t\t\tqueryLength,\n\t\t\tkeyLength );\n\t}\n\n\tpublic static bool IsAttentionAllowed(\n\t\tstring attentionType,\n\t\tint windowSize,\n\t\tint query,\n\t\tint key,\n\t\tint queryLength,\n\t\tint keyLength )\n\t{\n\t\tif ( attentionType != \"global\" && attentionType != \"local\" )\n\t\t{\n\t\t\tthrow new InvalidOperationException(\n\t\t\t\t$\"[LLM:ERROR] GPT-Neo attention type must be global or local, \" +\n\t\t\t\t$\"found '{attentionType}'.\" );\n\t\t}\n\t\tif ( attentionType == \"local\" && windowSize <= 0 )\n\t\t{\n\t\t\tthrow new InvalidOperationException(\n\t\t\t\t$\"[LLM:ERROR] Local GPT-Neo attention window must be positive, \" +\n\t\t\t\t$\"found {windowSize}.\" );\n\t\t}\n\t\tif ( queryLength <= 0 || keyLength <= 0 || queryLength > keyLength ||\n\t\t\tquery < 0 || query >= queryLength || key < 0 || key >= keyLength )\n\t\t{\n\t\t\tthrow new IndexOutOfRangeException(\n\t\t\t\t$\"[LLM:ERROR] Attention mask index query={query}, key={key} is invalid \" +\n\t\t\t\t$\"for query_length={queryLength}, key_length={keyLength}. \" +\n\t\t\t\t\"The verified source slice requires 0 < query_length <= key_length.\" );\n\t\t}\n\n\t\tint absoluteQuery = keyLength - queryLength + query;\n\t\tbool causal = key <= absoluteQuery;\n\t\treturn attentionType == \"global\"\n\t\t\t? causal\n\t\t\t: causal && absoluteQuery - key < windowSize;\n\t}\n\n\tpublic static Tensor ApplyLayer0AttentionMask(\n\t\tTensor unmaskedScores,\n\t\tstring attentionType,\n\t\tfloat maskedSentinel,\n\t\tstring outputName )\n\t{\n\t\treturn ApplyAttentionMask(\n\t\t\tunmaskedScores,\n\t\t\tattentionType,\n\t\t\twindowSize: 0,\n\t\t\tmaskedSentinel,\n\t\t\toutputName );\n\t}\n\n\tpublic static Tensor ApplyAttentionMask(\n\t\tTensor unmaskedScores,\n\t\tstring attentionType,\n\t\tint windowSize,\n\t\tfloat maskedSentinel,\n\t\tstring outputName )\n\t{\n\t\tif ( unmaskedScores is null )\n\t\t{\n\t\t\tthrow new ArgumentNullException( nameof( unmaskedScores ) );\n\t\t}\n\t\tif ( string.IsNullOrWhiteSpace( outputName ) )\n\t\t{\n\t\t\tthrow new ArgumentException(\n\t\t\t\t\"[LLM:ERROR] Masked attention-score output name cannot be empty.\",\n\t\t\t\tnameof( outputName ) );\n\t\t}\n\t\tif ( unmaskedScores.Rank != 3 || unmaskedScores.Shape[0] <= 0 ||\n\t\t\tunmaskedScores.Shape[1] <= 0 || unmaskedScores.Shape[2] <= 0 ||\n\t\t\tunmaskedScores.Shape[1] > unmaskedScores.Shape[2] )\n\t\t{\n\t\t\tthrow new InvalidOperationException(\n\t\t\t\t$\"[LLM:ERROR] {outputName} expected unmasked scores shape \" +\n\t\t\t\t$\"[head,query,key] with 0 < query <= key, found \" +\n\t\t\t\t$\"{unmaskedScores.ShapeText}.\" );\n\t\t}\n\t\tif ( !float.IsFinite( maskedSentinel ) || !(maskedSentinel < 0) )\n\t\t{\n\t\t\tthrow new InvalidOperationException(\n\t\t\t\t$\"[LLM:ERROR] {outputName} mask sentinel must be finite and negative, \" +\n\t\t\t\t$\"found {maskedSentinel:G9}.\" );\n\t\t}\n\n\t\tint headCount = unmaskedScores.Shape[0];\n\t\tint queryLength = unmaskedScores.Shape[1];\n\t\tint keyLength = unmaskedScores.Shape[2];\n\t\tfloat[] output = new float[unmaskedScores.Data.Length];\n\t\tfor ( int head = 0; head < headCount; head++ )\n\t\t{\n\t\t\tfor ( int query = 0; query < queryLength; query++ )\n\t\t\t{\n\t\t\t\tfor ( int key = 0; key < keyLength; key++ )\n\t\t\t\t{\n\t\t\t\t\tint index = (head * queryLength + query) * keyLength + key;\n\t\t\t\t\tbool allowed = IsAttentionAllowed(\n\t\t\t\t\t\tattentionType,\n\t\t\t\t\t\twindowSize,\n\t\t\t\t\t\tquery,\n\t\t\t\t\t\tkey,\n\t\t\t\t\t\tqueryLength,\n\t\t\t\t\t\tkeyLength );\n\t\t\t\t\toutput[index] = allowed ? unmaskedScores.Data[index] : maskedSentinel;\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\n\t\treturn new Tensor(\n\t\t\toutputName,\n\t\t\tnew[] { headCount, queryLength, keyLength },\n\t\t\toutput );\n\t}\n\n\tpublic static Tensor ComputeAttentionProbabilities(\n\t\tTensor maskedScores,\n\t\tstring outputName )\n\t{\n\t\tif ( maskedScores is null )\n\t\t{\n\t\t\tthrow new ArgumentNullException( nameof( maskedScores ) );\n\t\t}\n\t\tif ( string.IsNullOrWhiteSpace( outputName ) )\n\t\t{\n\t\t\tthrow new ArgumentException(\n\t\t\t\t\"[LLM:ERROR] Attention-probability output name cannot be empty.\",\n\t\t\t\tnameof( outputName ) );\n\t\t}\n\t\tif ( maskedScores.Rank != 3 || maskedScores.Shape[0] <= 0 ||\n\t\t\tmaskedScores.Shape[1] <= 0 || maskedScores.Shape[2] <= 0 )\n\t\t{\n\t\t\tthrow new InvalidOperationException(\n\t\t\t\t$\"[LLM:ERROR] {outputName} expected masked scores shape \" +\n\t\t\t\t$\"[head,query,key] with positive dimensions, found \" +\n\t\t\t\t$\"{maskedScores.ShapeText}.\" );\n\t\t}\n\n\t\tint headCount = maskedScores.Shape[0];\n\t\tint queryLength = maskedScores.Shape[1];\n\t\tint keyLength = maskedScores.Shape[2];\n\t\tfloat[] output = new float[maskedScores.Data.Length];\n\t\tfor ( int head = 0; head < headCount; head++ )\n\t\t{\n\t\t\tfor ( int query = 0; query < queryLength; query++ )\n\t\t\t{\n\t\t\t\tint row = (head * queryLength + query) * keyLength;\n\t\t\t\tfloat rowMaximum = float.NegativeInfinity;\n\t\t\t\tfor ( int key = 0; key < keyLength; key++ )\n\t\t\t\t{\n\t\t\t\t\tfloat value = maskedScores.Data[row + key];\n\t\t\t\t\tif ( !float.IsFinite( value ) )\n\t\t\t\t\t{\n\t\t\t\t\t\tthrow new InvalidOperationException(\n\t\t\t\t\t\t\t$\"[LLM:ERROR] {outputName} input contains a non-finite score \" +\n\t\t\t\t\t\t\t$\"at [{head},{query},{key}]: {value}.\" );\n\t\t\t\t\t}\n\t\t\t\t\trowMaximum = MathF.Max( rowMaximum, value );\n\t\t\t\t}\n\n\t\t\t\tfloat exponentialSum = 0;\n\t\t\t\tfor ( int key = 0; key < keyLength; key++ )\n\t\t\t\t{\n\t\t\t\t\tfloat shifted = maskedScores.Data[row + key] - rowMaximum;\n\t\t\t\t\tfloat exponential = MathF.Exp( shifted );\n\t\t\t\t\tif ( !float.IsFinite( exponential ) || exponential < 0 )\n\t\t\t\t\t{\n\t\t\t\t\t\tthrow new InvalidOperationException(\n\t\t\t\t\t\t\t$\"[LLM:ERROR] {outputName} produced invalid exp at \" +\n\t\t\t\t\t\t\t$\"[{head},{query},{key}]: input={maskedScores.Data[row + key]:G9}, \" +\n\t\t\t\t\t\t\t$\"maximum={rowMaximum:G9}, shifted={shifted:G9}, \" +\n\t\t\t\t\t\t\t$\"exp={exponential:G9}.\" );\n\t\t\t\t\t}\n\t\t\t\t\toutput[row + key] = exponential;\n\t\t\t\t\texponentialSum += exponential;\n\t\t\t\t}\n\n\t\t\t\tif ( !(exponentialSum > 0) || !float.IsFinite( exponentialSum ) )\n\t\t\t\t{\n\t\t\t\t\tthrow new InvalidOperationException(\n\t\t\t\t\t\t$\"[LLM:ERROR] {outputName} row [{head},{query}] produced invalid \" +\n\t\t\t\t\t\t$\"exponential sum {exponentialSum:G9}.\" );\n\t\t\t\t}\n\n\t\t\t\tfloat inverseSum = 1.0f / exponentialSum;\n\t\t\t\tfor ( int key = 0; key < keyLength; key++ )\n\t\t\t\t{\n\t\t\t\t\toutput[row + key] *= inverseSum;\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\n\t\treturn new Tensor(\n\t\t\toutputName,\n\t\t\tnew[] { headCount, queryLength, keyLength },\n\t\t\toutput );\n\t}\n\n\tpublic static Tensor ComputeAttentionContextHeads(\n\t\tTensor probabilities,\n\t\tTensor valueHeads,\n\t\tstring outputName )\n\t{\n\t\tif ( probabilities is null )\n\t\t{\n\t\t\tthrow new ArgumentNullException( nameof( probabilities ) );\n\t\t}\n\t\tif ( valueHeads is null )\n\t\t{\n\t\t\tthrow new ArgumentNullException( nameof( valueHeads ) );\n\t\t}\n\t\tif ( string.IsNullOrWhiteSpace( outputName ) )\n\t\t{\n\t\t\tthrow new ArgumentException(\n\t\t\t\t\"[LLM:ERROR] Attention-context output name cannot be empty.\",\n\t\t\t\tnameof( outputName ) );\n\t\t}\n\t\tif ( probabilities.Rank != 3 || probabilities.Shape[0] <= 0 ||\n\t\t\tprobabilities.Shape[1] <= 0 || probabilities.Shape[2] <= 0 )\n\t\t{\n\t\t\tthrow new InvalidOperationException(\n\t\t\t\t$\"[LLM:ERROR] {outputName} expected probabilities shape \" +\n\t\t\t\t$\"[head,query,key] with positive dimensions, found \" +\n\t\t\t\t$\"{probabilities.ShapeText}.\" );\n\t\t}\n\t\tif ( valueHeads.Rank != 3 || valueHeads.Shape[0] <= 0 ||\n\t\t\tvalueHeads.Shape[1] <= 0 || valueHeads.Shape[2] <= 0 )\n\t\t{\n\t\t\tthrow new InvalidOperationException(\n\t\t\t\t$\"[LLM:ERROR] {outputName} expected value shape \" +\n\t\t\t\t$\"[head,key,component] with positive dimensions, found \" +\n\t\t\t\t$\"{valueHeads.ShapeText}.\" );\n\t\t}\n\t\tif ( probabilities.Shape[0] != valueHeads.Shape[0] ||\n\t\t\tprobabilities.Shape[2] != valueHeads.Shape[1] )\n\t\t{\n\t\t\tthrow new InvalidOperationException(\n\t\t\t\t$\"[LLM:ERROR] {outputName} incompatible probabilities/value shapes: \" +\n\t\t\t\t$\"probabilities={probabilities.ShapeText} [head,query,key], \" +\n\t\t\t\t$\"V={valueHeads.ShapeText} [head,key,component].\" );\n\t\t}\n\n\t\tint headCount = probabilities.Shape[0];\n\t\tint queryLength = probabilities.Shape[1];\n\t\tint keyLength = probabilities.Shape[2];\n\t\tint headDimension = valueHeads.Shape[2];\n\t\tfloat[] output = new float[headCount * queryLength * headDimension];\n\t\tfor ( int head = 0; head < headCount; head++ )\n\t\t{\n\t\t\tfor ( int query = 0; query < queryLength; query++ )\n\t\t\t{\n\t\t\t\tint probabilityRow = (head * queryLength + query) * keyLength;\n\t\t\t\tfor ( int component = 0; component < headDimension; component++ )\n\t\t\t\t{\n\t\t\t\t\tfloat sum = 0;\n\t\t\t\t\tfor ( int key = 0; key < keyLength; key++ )\n\t\t\t\t\t{\n\t\t\t\t\t\tfloat probability = probabilities.Data[probabilityRow + key];\n\t\t\t\t\t\tint valueIndex = (head * keyLength + key) * headDimension + component;\n\t\t\t\t\t\tfloat value = valueHeads.Data[valueIndex];\n\t\t\t\t\t\tif ( !float.IsFinite( probability ) || !float.IsFinite( value ) )\n\t\t\t\t\t\t{\n\t\t\t\t\t\t\tthrow new InvalidOperationException(\n\t\t\t\t\t\t\t\t$\"[LLM:ERROR] {outputName} received non-finite input at \" +\n\t\t\t\t\t\t\t\t$\"head={head}, query={query}, key={key}, component={component}: \" +\n\t\t\t\t\t\t\t\t$\"probability={probability:G9}, V={value:G9}.\" );\n\t\t\t\t\t\t}\n\t\t\t\t\t\tsum += probability * value;\n\t\t\t\t\t}\n\n\t\t\t\t\tif ( !float.IsFinite( sum ) )\n\t\t\t\t\t{\n\t\t\t\t\t\tthrow new InvalidOperationException(\n\t\t\t\t\t\t\t$\"[LLM:ERROR] {outputName} produced non-finite context at \" +\n\t\t\t\t\t\t\t$\"[{head},{query},{component}]: {sum:G9}.\" );\n\t\t\t\t\t}\n\t\t\t\t\tint outputIndex =\n\t\t\t\t\t\t(head * queryLength + query) * headDimension + component;\n\t\t\t\t\toutput[outputIndex] = sum;\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\n\t\treturn new Tensor(\n\t\t\toutputName,\n\t\t\tnew[] { headCount, queryLength, headDimension },\n\t\t\toutput );\n\t}\n\n\tpublic static string DescribeAttentionContextElement(\n\t\tTensor probabilities,\n\t\tTensor valueHeads,\n\t\tTensor context,\n\t\tint head,\n\t\tint query,\n\t\tint component,\n\t\tfloat expected )\n\t{\n\t\tif ( probabilities is null || valueHeads is null || context is null ||\n\t\t\tprobabilities.Rank != 3 || valueHeads.Rank != 3 || context.Rank != 3 ||\n\t\t\tprobabilities.Shape[0] != valueHeads.Shape[0] ||\n\t\t\tprobabilities.Shape[0] != context.Shape[0] ||\n\t\t\tprobabilities.Shape[1] != context.Shape[1] ||\n\t\t\tprobabilities.Shape[2] != valueHeads.Shape[1] ||\n\t\t\tvalueHeads.Shape[2] != context.Shape[2] )\n\t\t{\n\t\t\tthrow new InvalidOperationException(\n\t\t\t\t\"[LLM:ERROR] Attention-context diagnostic requires compatible \" +\n\t\t\t\t\"probability [head,query,key], V [head,key,component], and context \" +\n\t\t\t\t\"[head,query,component] tensors.\" );\n\t\t}\n\t\tif ( head < 0 || head >= context.Shape[0] ||\n\t\t\tquery < 0 || query >= context.Shape[1] ||\n\t\t\tcomponent < 0 || component >= context.Shape[2] )\n\t\t{\n\t\t\tthrow new IndexOutOfRangeException(\n\t\t\t\t$\"[LLM:ERROR] Attention-context diagnostic index \" +\n\t\t\t\t$\"[{head},{query},{component}] is outside {context.ShapeText}.\" );\n\t\t}\n\n\t\tint keyLength = probabilities.Shape[2];\n\t\tint headDimension = valueHeads.Shape[2];\n\t\tint probabilityRow = (head * probabilities.Shape[1] + query) * keyLength;\n\t\tfloat[] probabilityValues = new float[keyLength];\n\t\tfloat[] values = new float[keyLength];\n\t\tfloat[] products = new float[keyLength];\n\t\tfloat sum = 0;\n\t\tfor ( int key = 0; key < keyLength; key++ )\n\t\t{\n\t\t\tprobabilityValues[key] = probabilities.Data[probabilityRow + key];\n\t\t\tint valueIndex = (head * keyLength + key) * headDimension + component;\n\t\t\tvalues[key] = valueHeads.Data[valueIndex];\n\t\t\tproducts[key] = probabilityValues[key] * values[key];\n\t\t\tsum += products[key];\n\t\t}\n\n\t\tint contextIndex =\n\t\t\t(head * context.Shape[1] + query) * context.Shape[2] + component;\n\t\tfloat actual = context.Data[contextIndex];\n\t\tif ( BitConverter.SingleToInt32Bits( sum ) !=\n\t\t\tBitConverter.SingleToInt32Bits( actual ) )\n\t\t{\n\t\t\tthrow new InvalidOperationException(\n\t\t\t\t$\"[LLM:ERROR] Attention-context diagnostic recomputation differs at \" +\n\t\t\t\t$\"[{head},{query},{component}]: recomputed={sum:G9}, actual={actual:G9}.\" );\n\t\t}\n\n\t\treturn $\"head={head} query={query} component={component} \" +\n\t\t\t$\"probabilities={FormatFloatList( probabilityValues )} \" +\n\t\t\t$\"V={FormatFloatList( values )} products={FormatFloatList( products )} \" +\n\t\t\t$\"accumulated={actual:G9} python={expected:G9} \" +\n\t\t\t$\"abs_diff={MathF.Abs( actual - expected ):G12}\";\n\t}\n\n\tpublic static string DescribeAttentionSoftmaxRow(\n\t\tTensor maskedScores,\n\t\tTensor probabilities,\n\t\tTensor expectedProbabilities,\n\t\tint head,\n\t\tint query )\n\t{\n\t\tif ( maskedScores is null || probabilities is null || expectedProbabilities is null ||\n\t\t\tmaskedScores.Rank != 3 || probabilities.Rank != 3 || expectedProbabilities.Rank != 3 ||\n\t\t\tmaskedScores.Shape[0] != probabilities.Shape[0] ||\n\t\t\tmaskedScores.Shape[1] != probabilities.Shape[1] ||\n\t\t\tmaskedScores.Shape[2] != probabilities.Shape[2] ||\n\t\t\tmaskedScores.Shape[0] != expectedProbabilities.Shape[0] ||\n\t\t\tmaskedScores.Shape[1] != expectedProbabilities.Shape[1] ||\n\t\t\tmaskedScores.Shape[2] != expectedProbabilities.Shape[2] )\n\t\t{\n\t\t\tthrow new InvalidOperationException(\n\t\t\t\t\"[LLM:ERROR] Softmax row diagnostic requires matching rank-3 tensors.\" );\n\t\t}\n\t\tif ( head < 0 || head >= maskedScores.Shape[0] ||\n\t\t\tquery < 0 || query >= maskedScores.Shape[1] )\n\t\t{\n\t\t\tthrow new IndexOutOfRangeException(\n\t\t\t\t$\"[LLM:ERROR] Softmax row diagnostic index [{head},{query}] is outside \" +\n\t\t\t\t$\"{maskedScores.ShapeText}.\" );\n\t\t}\n\n\t\tint keyLength = maskedScores.Shape[2];\n\t\tint row = (head * maskedScores.Shape[1] + query) * keyLength;\n\t\tfloat rowMaximum = float.NegativeInfinity;\n\t\tfor ( int key = 0; key < keyLength; key++ )\n\t\t{\n\t\t\trowMaximum = MathF.Max( rowMaximum, maskedScores.Data[row + key] );\n\t\t}\n\n\t\tfloat[] inputs = new float[keyLength];\n\t\tfloat[] shifted = new float[keyLength];\n\t\tfloat[] exponentials = new float[keyLength];\n\t\tfloat[] actual = new float[keyLength];\n\t\tfloat[] expected = new float[keyLength];\n\t\tfloat exponentialSum = 0;\n\t\tfloat probabilitySum = 0;\n\t\tfor ( int key = 0; key < keyLength; key++ )\n\t\t{\n\t\t\tinputs[key] = maskedScores.Data[row + key];\n\t\t\tshifted[key] = inputs[key] - rowMaximum;\n\t\t\texponentials[key] = MathF.Exp( shifted[key] );\n\t\t\texponentialSum += exponentials[key];\n\t\t\tactual[key] = probabilities.Data[row + key];\n\t\t\texpected[key] = expectedProbabilities.Data[row + key];\n\t\t\tprobabilitySum += actual[key];\n\t\t}\n\n\t\treturn $\"head={head} query={query} input={FormatFloatList( inputs )} \" +\n\t\t\t$\"max={rowMaximum:G9} shifted={FormatFloatList( shifted )} \" +\n\t\t\t$\"exp={FormatFloatList( exponentials )} exp_sum={exponentialSum:G9} \" +\n\t\t\t$\"actual={FormatFloatList( actual )} python={FormatFloatList( expected )} \" +\n\t\t\t$\"row_sum={probabilitySum:G9}\";\n\t}\n\n\tpublic static string DescribeAttentionMaskApplication(\n\t\tTensor unmaskedScores,\n\t\tTensor maskedScores,\n\t\tstring attentionType,\n\t\tfloat maskedSentinel,\n\t\tint head,\n\t\tint query,\n\t\tint key )\n\t{\n\t\tif ( unmaskedScores is null || maskedScores is null ||\n\t\t\tunmaskedScores.Rank != 3 || maskedScores.Rank != 3 ||\n\t\t\tunmaskedScores.Shape[0] != maskedScores.Shape[0] ||\n\t\t\tunmaskedScores.Shape[1] != maskedScores.Shape[1] ||\n\t\t\tunmaskedScores.Shape[2] != maskedScores.Shape[2] )\n\t\t{\n\t\t\tthrow new InvalidOperationException(\n\t\t\t\t\"[LLM:ERROR] Mask diagnostic requires matching rank-3 unmasked/masked tensors.\" );\n\t\t}\n\n\t\tint queryLength = unmaskedScores.Shape[1];\n\t\tint keyLength = unmaskedScores.Shape[2];\n\t\tif ( head < 0 || head >= unmaskedScores.Shape[0] )\n\t\t{\n\t\t\tthrow new IndexOutOfRangeException(\n\t\t\t\t$\"[LLM:ERROR] Mask diagnostic head {head} is outside \" +\n\t\t\t\t$\"{unmaskedScores.ShapeText}.\" );\n\t\t}\n\t\tbool allowed = IsLayer0AttentionAllowed(\n\t\t\tattentionType,\n\t\t\tquery,\n\t\t\tkey,\n\t\t\tqueryLength,\n\t\t\tkeyLength );\n\t\tint index = (head * queryLength + query) * keyLength + key;\n\t\tfloat unmasked = unmaskedScores.Data[index];\n\t\tfloat masked = maskedScores.Data[index];\n\t\treturn $\"head={head} query={query} key={key} allowed={allowed} \" +\n\t\t\t$\"unmasked={unmasked:G9} masked={masked:G9} \" +\n\t\t\t$\"masked_bits=0x{BitConverter.SingleToInt32Bits( masked ):X8} \" +\n\t\t\t$\"expected_sentinel={maskedSentinel:G9} \" +\n\t\t\t$\"sentinel_bits=0x{BitConverter.SingleToInt32Bits( maskedSentinel ):X8}\";\n\t}\n\n\tpublic static string DescribeAttentionScore(\n\t\tTensor queryHeads,\n\t\tTensor keyHeads,\n\t\tTensor scores,\n\t\tint head,\n\t\tint query,\n\t\tint key,\n\t\tfloat scale,\n\t\tfloat expected )\n\t{\n\t\tif ( queryHeads is null || keyHeads is null || scores is null ||\n\t\t\tqueryHeads.Rank != 3 || keyHeads.Rank != 3 || scores.Rank != 3 )\n\t\t{\n\t\t\tthrow new InvalidOperationException(\n\t\t\t\t\"[LLM:ERROR] Attention-score diagnostic requires rank-3 Q, K, and score tensors.\" );\n\t\t}\n\t\tint headDimension = queryHeads.Shape[2];\n\t\tif ( headDimension != 4 || keyHeads.Shape[2] != headDimension ||\n\t\t\tqueryHeads.Shape[0] != keyHeads.Shape[0] ||\n\t\t\tscores.Shape[0] != queryHeads.Shape[0] ||\n\t\t\tscores.Shape[1] != queryHeads.Shape[1] ||\n\t\t\tscores.Shape[2] != keyHeads.Shape[1] )\n\t\t{\n\t\t\tthrow new InvalidOperationException(\n\t\t\t\t$\"[LLM:ERROR] Attention-score diagnostic expected compatible Q/K/scores with \" +\n\t\t\t\t$\"head dimension 4, found Q={queryHeads.ShapeText}, K={keyHeads.ShapeText}, \" +\n\t\t\t\t$\"scores={scores.ShapeText}.\" );\n\t\t}\n\t\tif ( head < 0 || head >= scores.Shape[0] || query < 0 || query >= scores.Shape[1] ||\n\t\t\tkey < 0 || key >= scores.Shape[2] )\n\t\t{\n\t\t\tthrow new IndexOutOfRangeException(\n\t\t\t\t$\"[LLM:ERROR] Attention-score diagnostic index [{head},{query},{key}] \" +\n\t\t\t\t$\"is outside {scores.ShapeText}.\" );\n\t\t}\n\n\t\tint queryRow = (head * queryHeads.Shape[1] + query) * headDimension;\n\t\tint keyRow = (head * keyHeads.Shape[1] + key) * headDimension;\n\t\tfloat q0 = queryHeads.Data[queryRow];\n\t\tfloat q1 = queryHeads.Data[queryRow + 1];\n\t\tfloat q2 = queryHeads.Data[queryRow + 2];\n\t\tfloat q3 = queryHeads.Data[queryRow + 3];\n\t\tfloat k0 = keyHeads.Data[keyRow];\n\t\tfloat k1 = keyHeads.Data[keyRow + 1];\n\t\tfloat k2 = keyHeads.Data[keyRow + 2];\n\t\tfloat k3 = keyHeads.Data[keyRow + 3];\n\t\tfloat p0 = q0 * k0;\n\t\tfloat p1 = q1 * k1;\n\t\tfloat p2 = q2 * k2;\n\t\tfloat p3 = q3 * k3;\n\t\tfloat rawSum = ((p0 + p1) + p2) + p3;\n\t\tfloat recomputed = scale == 1.0f ? rawSum : rawSum * scale;\n\t\tint scoreIndex = (head * scores.Shape[1] + query) * scores.Shape[2] + key;\n\t\tfloat actual = scores.Data[scoreIndex];\n\t\tif ( BitConverter.SingleToInt32Bits( recomputed ) !=\n\t\t\tBitConverter.SingleToInt32Bits( actual ) )\n\t\t{\n\t\t\tthrow new InvalidOperationException(\n\t\t\t\t$\"[LLM:ERROR] Attention-score diagnostic recomputation differs at \" +\n\t\t\t\t$\"[{head},{query},{key}]: recomputed={recomputed:G9}, actual={actual:G9}.\" );\n\t\t}\n\n\t\treturn $\"head={head} query={query} key={key} \" +\n\t\t\t$\"q=[{q0:G9},{q1:G9},{q2:G9},{q3:G9}] \" +\n\t\t\t$\"k=[{k0:G9},{k1:G9},{k2:G9},{k3:G9}] \" +\n\t\t\t$\"products=[{p0:G9},{p1:G9},{p2:G9},{p3:G9}] \" +\n\t\t\t$\"raw_sum={rawSum:G9} scale={scale:G9} final={actual:G9} expected={expected:G9}\";\n\t}\n\n\tprivate static string FormatFloatList( IReadOnlyList<float> values )\n\t{\n\t\tstring[] formatted = new string[values.Count];\n\t\tfor ( int index = 0; index < values.Count; index++ )\n\t\t{\n\t\t\tformatted[index] = values[index].ToString( \"G9\" );\n\t\t}\n\t\treturn $\"[{string.Join( \",\", formatted )}]\";\n\t}\n}\n"
},
{
"Ident": "jeffskitchen.llm_poc",
"Path": "Llm/TinyStoriesGreedyGenerator.cs",
"FileName": "TinyStoriesGreedyGenerator.cs",
"PackageType": "game",
"CodeKind": "Game",
"AssetVersionId": 342521,
"Code": "using Sandbox.Diagnostics;\n\nnamespace LlmPoc.Llm;\n\npublic sealed class GreedyGenerationStepResult\n{\n\tpublic int Step { get; init; }\n\tpublic int InputSequenceLength { get; init; }\n\tpublic int NewTokenPosition { get; init; }\n\tpublic int TokenId { get; init; }\n\tpublic string DecodedToken { get; init; }\n\tpublic float Top1Logit { get; init; }\n\tpublic int Top2TokenId { get; init; }\n\tpublic float Top2Logit { get; init; }\n\tpublic float Top1Top2Margin { get; init; }\n\tpublic LogitRank[] TopFive { get; init; }\n\tpublic bool EosReached { get; init; }\n\tpublic double ForwardMilliseconds { get; init; }\n\tpublic double LmHeadMilliseconds { get; init; }\n\tpublic double StepMilliseconds { get; init; }\n\tpublic double[] LayerMilliseconds { get; init; }\n}\n\npublic sealed class GreedyGenerationStepObservation\n{\n\tpublic GreedyGenerationStepResult Step { get; init; }\n\tpublic int[] InputTokenIds { get; init; }\n\tpublic TinyStoriesModelForwardResult Forward { get; init; }\n}\n\npublic sealed class GreedyGenerationResult\n{\n\tpublic int[] PromptTokenIds { get; init; }\n\tpublic int[] GeneratedTokenIds { get; init; }\n\tpublic int[] FullSequenceTokenIds { get; init; }\n\tpublic string GeneratedText { get; init; }\n\tpublic string StopReason { get; init; }\n\tpublic bool EosReached { get; init; }\n\tpublic GreedyGenerationStepResult[] Steps { get; init; }\n\tpublic double TotalForwardMilliseconds { get; init; }\n\tpublic double TotalGenerationMilliseconds { get; init; }\n}\n\n/// <summary>\n/// Deterministic correctness baseline: every new token recomputes the complete\n/// model over the complete current context. No K/V state is retained.\n/// </summary>\npublic static class TinyStoriesGreedyGenerator\n{\n\tpublic static GreedyGenerationResult Generate(\n\t\tSboxLlmModel model,\n\t\tTinyStoriesConfig config,\n\t\tGpt2ByteBpeTokenizer tokenizer,\n\t\tIReadOnlyList<int> promptTokenIds,\n\t\tint maxNewTokens,\n\t\tAction<GreedyGenerationStepObservation> observer = null,\n\t\tbool logLifecycle = true )\n\t{\n\t\tif ( model is null ) throw new ArgumentNullException( nameof( model ) );\n\t\tif ( config is null ) throw new ArgumentNullException( nameof( config ) );\n\t\tif ( tokenizer is null ) throw new ArgumentNullException( nameof( tokenizer ) );\n\t\tif ( promptTokenIds is null ) throw new ArgumentNullException( nameof( promptTokenIds ) );\n\t\tif ( promptTokenIds.Count == 0 )\n\t\t{\n\t\t\tthrow new ArgumentException(\n\t\t\t\t\"[LLM:ERROR] Greedy generation requires at least one prompt token.\",\n\t\t\t\tnameof( promptTokenIds ) );\n\t\t}\n\t\tif ( maxNewTokens <= 0 )\n\t\t{\n\t\t\tthrow new ArgumentOutOfRangeException(\n\t\t\t\tnameof( maxNewTokens ), maxNewTokens,\n\t\t\t\t\"[LLM:ERROR] Greedy generation maxNewTokens must be positive.\" );\n\t\t}\n\t\tif ( promptTokenIds.Count > config.MaximumPositions - maxNewTokens )\n\t\t{\n\t\t\tthrow new InvalidOperationException(\n\t\t\t\t$\"[LLM:ERROR] Greedy generation prompt length {promptTokenIds.Count} plus \" +\n\t\t\t\t$\"maxNewTokens {maxNewTokens} would exceed maximum context length \" +\n\t\t\t\t$\"{config.MaximumPositions}; context truncation is disabled.\" );\n\t\t}\n\n\t\tint[] promptCopy = promptTokenIds.ToArray();\n\t\tList<int> context = new( promptCopy.Length + maxNewTokens );\n\t\tcontext.AddRange( promptCopy );\n\t\tList<int> generated = new( maxNewTokens );\n\t\tList<GreedyGenerationStepResult> steps = new( maxNewTokens );\n\t\tdouble totalForwardMilliseconds = 0;\n\t\tdouble totalGenerationMilliseconds = 0;\n\t\tbool eosReached = false;\n\n\t\tif ( logLifecycle )\n\t\t{\n\t\t\tLlmLog.Info(\n\t\t\t\t\"GEN\",\n\t\t\t\t$\"starting prompt_tokens={promptCopy.Length} max_new_tokens={maxNewTokens} \" +\n\t\t\t\t$\"maximum_context={config.MaximumPositions} strategy=greedy \" +\n\t\t\t\t\"full_recompute=true kv_cache=false\" );\n\t\t}\n\n\t\tfor ( int stepIndex = 0; stepIndex < maxNewTokens; stepIndex++ )\n\t\t{\n\t\t\tFastTimer stepTimer = FastTimer.StartNew();\n\t\t\tTinyStoriesModelForwardResult forward =\n\t\t\t\tTinyStoriesModelForward.ForwardLastTokenLogits( model, config, context );\n\t\t\tint tokenId = TinyStoriesModelHead.ArgmaxFinite(\n\t\t\t\tforward.Logits.Data, $\"generation.step{stepIndex}.logits\" );\n\t\t\tLogitRank[] topFive = TinyStoriesModelHead.TopKFinite(\n\t\t\t\tforward.Logits.Data, 5, $\"generation.step{stepIndex}.logits\" );\n\t\t\tstring decodedToken = tokenizer.Decode( new[] { tokenId } );\n\t\t\tfloat margin = topFive[0].Logit - topFive[1].Logit;\n\t\t\tbool stepEos = tokenId == config.EosTokenId;\n\t\t\tGreedyGenerationStepResult step = new()\n\t\t\t{\n\t\t\t\tStep = stepIndex,\n\t\t\t\tInputSequenceLength = context.Count,\n\t\t\t\tNewTokenPosition = context.Count,\n\t\t\t\tTokenId = tokenId,\n\t\t\t\tDecodedToken = decodedToken,\n\t\t\t\tTop1Logit = topFive[0].Logit,\n\t\t\t\tTop2TokenId = topFive[1].TokenId,\n\t\t\t\tTop2Logit = topFive[1].Logit,\n\t\t\t\tTop1Top2Margin = margin,\n\t\t\t\tTopFive = topFive,\n\t\t\t\tEosReached = stepEos,\n\t\t\t\tForwardMilliseconds = forward.TotalMilliseconds,\n\t\t\t\tLmHeadMilliseconds = forward.LmHeadMilliseconds,\n\t\t\t\tStepMilliseconds = stepTimer.ElapsedMilliSeconds,\n\t\t\t\tLayerMilliseconds = forward.LayerMilliseconds\n\t\t\t};\n\n\t\t\tGreedyGenerationStepObservation observation = new()\n\t\t\t{\n\t\t\t\tStep = step,\n\t\t\t\tInputTokenIds = context.ToArray(),\n\t\t\t\tForward = forward\n\t\t\t};\n\t\t\tobserver?.Invoke( observation );\n\t\t\tif ( observer is null && logLifecycle )\n\t\t\t{\n\t\t\t\tLlmLog.Info(\n\t\t\t\t\t\"GEN\",\n\t\t\t\t\t$\"step={stepIndex} context={context.Count} token={tokenId} \" +\n\t\t\t\t\t$\"piece='{EscapeVisible( decodedToken )}' margin={margin:G9} \" +\n\t\t\t\t\t$\"forward_ms={forward.TotalMilliseconds:N4}\" );\n\t\t\t}\n\t\t\tLlmLog.Trace(\n\t\t\t\t\"GEN\",\n\t\t\t\t$\"step={stepIndex} sequence={context.Count} new_token_position={context.Count} \" +\n\t\t\t\t$\"top5=[{string.Join( \",\", topFive.Select( item => $\"{item.TokenId}:{item.Logit:G9}\" ) )}]\" );\n\n\t\t\tcontext.Add( tokenId );\n\t\t\tgenerated.Add( tokenId );\n\t\t\tsteps.Add( step );\n\t\t\ttotalForwardMilliseconds += forward.TotalMilliseconds;\n\t\t\ttotalGenerationMilliseconds += step.StepMilliseconds;\n\t\t\tif ( stepEos )\n\t\t\t{\n\t\t\t\teosReached = true;\n\t\t\t\tbreak;\n\t\t\t}\n\t\t}\n\n\t\tfor ( int index = 0; index < promptCopy.Length; index++ )\n\t\t{\n\t\t\tif ( promptCopy[index] != promptTokenIds[index] )\n\t\t\t{\n\t\t\t\tthrow new InvalidOperationException(\n\t\t\t\t\t$\"[LLM:ERROR] Greedy generation mutated caller prompt token {index}.\" );\n\t\t\t}\n\t\t}\n\n\t\treturn new GreedyGenerationResult\n\t\t{\n\t\t\tPromptTokenIds = promptCopy,\n\t\t\tGeneratedTokenIds = generated.ToArray(),\n\t\t\tFullSequenceTokenIds = context.ToArray(),\n\t\t\tGeneratedText = tokenizer.Decode( generated ),\n\t\t\tStopReason = eosReached ? \"eos\" : \"max_new_tokens\",\n\t\t\tEosReached = eosReached,\n\t\t\tSteps = steps.ToArray(),\n\t\t\tTotalForwardMilliseconds = totalForwardMilliseconds,\n\t\t\tTotalGenerationMilliseconds = totalGenerationMilliseconds\n\t\t};\n\t}\n\n\tprivate static string EscapeVisible( string value )\n\t{\n\t\treturn value\n\t\t\t.Replace( \"\\\\\", \"\\\\\\\\\" )\n\t\t\t.Replace( \"\\r\", \"\\\\r\" )\n\t\t\t.Replace( \"\\n\", \"\\\\n\" )\n\t\t\t.Replace( \"\\t\", \"\\\\t\" )\n\t\t\t.Replace( \"'\", \"\\\\'\" );\n\t}\n}\n"
},
{
"Ident": "jeffskitchen.llm_poc",
"Path": "ui/button.cs.scss",
"FileName": "button.cs.scss",
"PackageType": "game",
"CodeKind": "Game",
"AssetVersionId": 342521,
"Code": ".button\r\n{\r\n\tposition: relative;\r\n\t\r\n\t> .button-right-column\r\n\t{\r\n\t\tflex-direction: column;\r\n\t}\r\n}\r\n\r\n// default menu position is below\r\n.button-hover-menu\r\n{\r\n\tposition: absolute;\r\n\ttop: 100%;\r\n\tflex-direction: column;\r\n\r\n\t&.hidden\r\n\t{\r\n\t\topacity: 0;\r\n\t\tpointer-events: none;\r\n\t}\r\n}"
},
{
"Ident": "jeffskitchen.llm_poc",
"Path": "ui/dropdown.cs.scss",
"FileName": "dropdown.cs.scss",
"PackageType": "game",
"CodeKind": "Game",
"AssetVersionId": 342521,
"Code": ".dropdown\r\n{\r\n\tgap: 2px;\r\n\tflex-grow: 1;\r\n\tcursor: pointer;\r\n\tjustify-content: flex-end;\r\n\talign-items: center;\r\n\tpadding: 0px 12px;\r\n\r\n\t.button-right-column\r\n\t{\r\n\t\tflex-grow: 1;\r\n\t}\r\n}\r\n"
},
{
"Ident": "jeffskitchen.llm_poc",
"Path": "ui/components/packagelist.razor.scss",
"FileName": "packagelist.razor.scss",
"PackageType": "game",
"CodeKind": "Game",
"AssetVersionId": 342521,
"Code": ".package-list\r\n{\r\n flex-shrink: 1;\r\n flex-wrap: wrap;\r\n flex-grow: 1;\r\n\r\n h1\r\n {\r\n width: 100%;\r\n margin-top: 50px;\r\n font-size: 40px;\r\n }\r\n\r\n PackageCard\r\n {\r\n &:hover\r\n {\r\n sound-in: \"ui.button.over\";\r\n }\r\n }\r\n\r\n VirtualGrid\r\n {\r\n width: 100%;\r\n height: 100%;\r\n\r\n .cell\r\n {\r\n \r\n }\r\n }\r\n}"
},
{
"Ident": "jeffskitchen.llm_poc",
"Path": "Llm/ReferenceFloatData.cs",
"FileName": "ReferenceFloatData.cs",
"PackageType": "game",
"CodeKind": "Game",
"AssetVersionId": 342521,
"Code": "namespace LlmPoc.Llm;\n\npublic static class ReferenceFloatData\n{\n\tpublic static float[] LoadFromMounted( string path, int expectedCount )\n\t{\n\t\tif ( !FileSystem.Mounted.FileExists( path ) )\n\t\t{\n\t\t\tthrow new InvalidOperationException(\n\t\t\t\t$\"[LLM:ERROR] Reference FP32 file '{path}' is missing from FileSystem.Mounted.\" );\n\t\t}\n\n\t\tbyte[] bytes = FileSystem.Mounted.ReadAllBytes( path ).ToArray();\n\t\tif ( bytes.Length % sizeof( float ) != 0 )\n\t\t{\n\t\t\tthrow new InvalidOperationException(\n\t\t\t\t$\"[LLM:ERROR] Reference FP32 file '{path}' has {bytes.Length:N0} bytes, \" +\n\t\t\t\t\"which is not divisible by four.\" );\n\t\t}\n\n\t\tint count = bytes.Length / sizeof( float );\n\t\tif ( count != expectedCount )\n\t\t{\n\t\t\tthrow new InvalidOperationException(\n\t\t\t\t$\"[LLM:ERROR] Reference FP32 file '{path}' expected {expectedCount:N0} values \" +\n\t\t\t\t$\"({expectedCount * sizeof( float ):N0} bytes), found {count:N0} values \" +\n\t\t\t\t$\"({bytes.Length:N0} bytes).\" );\n\t\t}\n\n\t\tfloat[] values = new float[count];\n\t\tfor ( int index = 0; index < count; index++ )\n\t\t{\n\t\t\tint offset = index * sizeof( float );\n\t\t\tuint bits = (uint)(\n\t\t\t\tbytes[offset]\n\t\t\t\t| (bytes[offset + 1] << 8)\n\t\t\t\t| (bytes[offset + 2] << 16)\n\t\t\t\t| (bytes[offset + 3] << 24) );\n\t\t\tvalues[index] = BitConverter.Int32BitsToSingle( unchecked( (int)bits ) );\n\t\t}\n\t\treturn values;\n\t}\n\n\tpublic static int ArgmaxFinite( ReadOnlySpan<float> values, string logicalName )\n\t{\n\t\tif ( values.Length == 0 )\n\t\t{\n\t\t\tthrow new ArgumentException(\n\t\t\t\t$\"[LLM:ERROR] {logicalName} cannot be argmaxed because it is empty.\" );\n\t\t}\n\n\t\tint bestIndex = 0;\n\t\tfloat bestValue = values[0];\n\t\tif ( !float.IsFinite( bestValue ) )\n\t\t{\n\t\t\tthrow new InvalidOperationException(\n\t\t\t\t$\"[LLM:ERROR] {logicalName} contains non-finite value {bestValue} at index 0.\" );\n\t\t}\n\n\t\tfor ( int index = 1; index < values.Length; index++ )\n\t\t{\n\t\t\tfloat value = values[index];\n\t\t\tif ( !float.IsFinite( value ) )\n\t\t\t{\n\t\t\t\tthrow new InvalidOperationException(\n\t\t\t\t\t$\"[LLM:ERROR] {logicalName} contains non-finite value {value} at index {index}.\" );\n\t\t\t}\n\t\t\tif ( value > bestValue )\n\t\t\t{\n\t\t\t\tbestValue = value;\n\t\t\t\tbestIndex = index;\n\t\t\t}\n\t\t}\n\t\treturn bestIndex;\n\t}\n}\n"
},
{
"Ident": "jeffskitchen.llm_poc",
"Path": "Llm/TinyStoriesModelForward.cs",
"FileName": "TinyStoriesModelForward.cs",
"PackageType": "game",
"CodeKind": "Game",
"AssetVersionId": 342521,
"Code": "using Sandbox.Diagnostics;\n\nnamespace LlmPoc.Llm;\n\npublic sealed class TinyStoriesModelForwardResult\n{\n\tpublic int SequenceLength { get; init; }\n\tpublic Tensor FinalLayerNorm { get; init; }\n\tpublic Tensor Logits { get; init; }\n\tpublic double EmbeddingMilliseconds { get; init; }\n\tpublic double[] LayerMilliseconds { get; init; }\n\tpublic string[] AttentionTypes { get; init; }\n\tpublic double AllLayersMilliseconds { get; init; }\n\tpublic double FinalLayerNormMilliseconds { get; init; }\n\tpublic double LmHeadMilliseconds { get; init; }\n\tpublic double TotalMilliseconds { get; init; }\n}\n\n/// <summary>\n/// Production-oriented full-context forward orchestration. It deliberately reuses\n/// the parity-proven embedding, transformer-layer, final-norm, and LM-head kernels.\n/// </summary>\npublic static class TinyStoriesModelForward\n{\n\tpublic static TinyStoriesModelForwardResult ForwardLastTokenLogits(\n\t\tSboxLlmModel model,\n\t\tTinyStoriesConfig config,\n\t\tIReadOnlyList<int> tokenIds,\n\t\tbool logDiagnostics = false )\n\t{\n\t\tif ( model is null ) throw new ArgumentNullException( nameof( model ) );\n\t\tif ( config is null ) throw new ArgumentNullException( nameof( config ) );\n\t\tif ( tokenIds is null ) throw new ArgumentNullException( nameof( tokenIds ) );\n\t\tif ( tokenIds.Count == 0 )\n\t\t{\n\t\t\tthrow new ArgumentException(\n\t\t\t\t\"[LLM:ERROR] Model forward requires at least one token.\", nameof( tokenIds ) );\n\t\t}\n\t\tif ( tokenIds.Count > config.MaximumPositions )\n\t\t{\n\t\t\tthrow new InvalidOperationException(\n\t\t\t\t$\"[LLM:ERROR] Model forward sequence length {tokenIds.Count} exceeds \" +\n\t\t\t\t$\"maximum context length {config.MaximumPositions}; context truncation is disabled.\" );\n\t\t}\n\t\tfor ( int index = 0; index < tokenIds.Count; index++ )\n\t\t{\n\t\t\tint tokenId = tokenIds[index];\n\t\t\tif ( tokenId < 0 || tokenId >= config.VocabularySize )\n\t\t\t{\n\t\t\t\tthrow new IndexOutOfRangeException(\n\t\t\t\t\t$\"[LLM:ERROR] Model forward token ID {tokenId} at index {index} is outside \" +\n\t\t\t\t\t$\"[0,{config.VocabularySize}).\" );\n\t\t\t}\n\t\t}\n\n\t\tFastTimer totalTimer = FastTimer.StartNew();\n\t\tFastTimer embeddingTimer = FastTimer.StartNew();\n\t\tTensor hiddenStates = TinyStoriesForwardStages.CombineEmbeddings(\n\t\t\tmodel, config, tokenIds, logDiagnostics );\n\t\tdouble embeddingMilliseconds = embeddingTimer.ElapsedMilliSeconds;\n\n\t\tdouble[] layerMilliseconds = new double[config.LayerCount];\n\t\tstring[] attentionTypes = new string[config.LayerCount];\n\t\tdouble allLayersMilliseconds = 0;\n\t\tfor ( int layerIndex = 0; layerIndex < config.LayerCount; layerIndex++ )\n\t\t{\n\t\t\tGptNeoLayerForwardResult layer = GptNeoTransformerLayer.Forward(\n\t\t\t\tmodel,\n\t\t\t\tconfig,\n\t\t\t\thiddenStates,\n\t\t\t\tlayerIndex,\n\t\t\t\tlogDiagnostics,\n\t\t\t\tvalidateInputMutation: false );\n\t\t\thiddenStates = layer.Output;\n\t\t\tlayerMilliseconds[layerIndex] = layer.ElapsedMilliseconds;\n\t\t\tattentionTypes[layerIndex] = layer.AttentionType;\n\t\t\tallLayersMilliseconds += layer.ElapsedMilliseconds;\n\t\t}\n\n\t\tFastTimer finalLayerNormTimer = FastTimer.StartNew();\n\t\tTensor finalLayerNorm = TinyStoriesForwardStages.ApplyFinalLayerNorm(\n\t\t\tmodel, config, hiddenStates, \"forward.final_ln\", logDiagnostics );\n\t\tdouble finalLayerNormMilliseconds = finalLayerNormTimer.ElapsedMilliSeconds;\n\n\t\tFastTimer lmHeadTimer = FastTimer.StartNew();\n\t\tTensor logits = TinyStoriesModelHead.ProjectLastPositionNoBias(\n\t\t\tfinalLayerNorm,\n\t\t\tmodel.GetRequiredTensor( \"lm_head.weight\" ),\n\t\t\t\"forward.final_logits_last_position\" );\n\t\tdouble lmHeadMilliseconds = lmHeadTimer.ElapsedMilliSeconds;\n\t\tlogits.RequireShape( config.VocabularySize );\n\n\t\tif ( logDiagnostics )\n\t\t{\n\t\t\tLlmLog.Trace(\n\t\t\t\t\"PERF\",\n\t\t\t\t$\"stage=model_forward sequence={tokenIds.Count} embeddings_ms={embeddingMilliseconds:N4} \" +\n\t\t\t\t$\"layers_ms={allLayersMilliseconds:N4} final_ln_ms={finalLayerNormMilliseconds:N4} \" +\n\t\t\t\t$\"lm_head_ms={lmHeadMilliseconds:N4} total_ms={totalTimer.ElapsedMilliSeconds:N4}\" );\n\t\t}\n\n\t\treturn new TinyStoriesModelForwardResult\n\t\t{\n\t\t\tSequenceLength = tokenIds.Count,\n\t\t\tFinalLayerNorm = finalLayerNorm,\n\t\t\tLogits = logits,\n\t\t\tEmbeddingMilliseconds = embeddingMilliseconds,\n\t\t\tLayerMilliseconds = layerMilliseconds,\n\t\t\tAttentionTypes = attentionTypes,\n\t\t\tAllLayersMilliseconds = allLayersMilliseconds,\n\t\t\tFinalLayerNormMilliseconds = finalLayerNormMilliseconds,\n\t\t\tLmHeadMilliseconds = lmHeadMilliseconds,\n\t\t\tTotalMilliseconds = totalTimer.ElapsedMilliSeconds\n\t\t};\n\t}\n}\n"
},
{
"Ident": "jeffskitchen.llm_poc",
"Path": "ui/controls/switchcontrol.razor.scss",
"FileName": "switchcontrol.razor.scss",
"PackageType": "game",
"CodeKind": "Game",
"AssetVersionId": 342521,
"Code": "\r\n.switchcontrol\r\n{\r\n flex-direction: row;\r\n width: 100px;\r\n min-height: 24px;\r\n align-items: center;\r\n cursor: pointer;\r\n\r\n .switch-frame\r\n {\r\n flex-grow: 0;\r\n flex-shrink: 1;\r\n width: 48px;\r\n height: 16px;\r\n background-color: #fff1;\r\n margin: 0px 5px;\r\n align-items: center;\r\n border-radius: 100px;\r\n transition: all 0.4s linear;\r\n\r\n .switch-inner\r\n {\r\n position: relative;\r\n flex-grow: 0;\r\n flex-shrink: 1;\r\n background-color: #999;\r\n width: 25px;\r\n height: 25px;\r\n border-radius: 100px;\r\n left: 20%;\r\n transform: translateX( -50% );\r\n transition: all 0.3s ease-out;\r\n }\r\n }\r\n\r\n &.active\r\n {\r\n .switch-frame\r\n {\r\n background-color: #fffa;\r\n }\r\n\r\n .switch-inner\r\n {\r\n left: 80%;\r\n background-color: #fff;\r\n }\r\n }\r\n}\r\n"
},
{
"Ident": "jeffskitchen.llm_poc",
"Path": "styles/form/_dropdown.scss",
"FileName": "_dropdown.scss",
"PackageType": "game",
"CodeKind": "Game",
"AssetVersionId": 342521,
"Code": "\r\n$primary: red !default;\r\n$primary-alt: white !default;\r\n\r\n$switch-padding: 6px !default;\r\n\r\n.button.popupbutton.dropdown\r\n{\r\n\tcursor: pointer;\r\n\ttransition: all .1s ease-out;\r\n\tposition: relative;\r\n\r\n\t> .dropdown_indicator\r\n\t{\r\n\t\tposition: absolute;\r\n\t\tright: 8px;\r\n\t}\r\n\r\n\t&.open\r\n\t{\r\n\t\tborder-bottom-left-radius: 1px;\r\n\t\tborder-bottom-right-radius: 1px;\r\n\t\ttransition: border-radius 0.2s ease-out;\r\n\t}\r\n}\r\n\r\nselect\r\n{\r\n\tmin-height: 40px;\r\n\r\n\t> option\r\n\t{\r\n\t\tdisplay: none;\r\n\t}\r\n}"
},
{
"Ident": "jeffskitchen.llm_poc",
"Path": "ui/menupanel.razor.scss",
"FileName": "menupanel.razor.scss",
"PackageType": "game",
"CodeKind": "Game",
"AssetVersionId": 342521,
"Code": "\r\nmenupanel\r\n{\r\n position: absolute;\r\n z-index: 1000;\r\n pointer-events: all;\r\n font-size: 12px;\r\n flex-shrink: 0;\r\n\r\n .background\r\n {\r\n position: absolute;\r\n left: -5000px;\r\n right: -5000px;\r\n top: -5000px;\r\n bottom: -5000px;\r\n }\r\n\r\n > .inner\r\n {\r\n min-width: 200px;\r\n min-height: 20px;\r\n flex-direction: column;\r\n font-family: Poppins;\r\n font-weight: bold;\r\n border-radius: 10px;\r\n box-shadow: 5px 5px 30px #000e;\r\n background-color: #2a2a2a;\r\n flex-shrink: 0;\r\n\r\n .spacer\r\n {\r\n height: 1px;\r\n background-color: #0005;\r\n }\r\n\r\n .option\r\n {\r\n color: #fffa;\r\n padding: 0px 8px;\r\n cursor: pointer;\r\n flex-shrink: 0;\r\n height: 32px;\r\n\r\n &:first-child\r\n {\r\n border-top-left-radius: 10px;\r\n border-top-right-radius: 10px;\r\n }\r\n\r\n &:last-child\r\n {\r\n border-bottom-left-radius: 10px;\r\n border-bottom-right-radius: 10px;\r\n }\r\n\r\n .icon\r\n {\r\n padding: 8px;\r\n font-family: Material Icons;\r\n justify-content: center;\r\n align-items: center;\r\n flex-shrink: 0;\r\n }\r\n\r\n .text\r\n {\r\n padding: 8px;\r\n flex-shrink: 0;\r\n }\r\n\r\n &:hover\r\n {\r\n background-color: #3472e6;\r\n color: #f5f8fe;\r\n }\r\n }\r\n }\r\n}\r\n"
}
]
}