GodotでもLLMを使いたい(C#とONNX編)

    Directory: E:\godot_projects\llm_onnx_test

Mode                 LastWriteTime         Length Name
----                 -------------         ------ ----
d----          2024/06/19    12:43                .godot
d----          2024/06/19    11:11                Models
-a---          2024/06/19    11:02             80 .gitattributes
-a---          2024/06/19    11:02             36 .gitignore
-a---          2024/06/19    11:55            764 chat_view.gd
-a---          2024/06/19    12:43           1185 chat_view.tscn
-a---          2024/06/19    11:02            949 icon.svg
-a---          2024/06/19    11:02            842 icon.svg.import
-a---          2024/06/19    11:08            504 llm_onnx_test.csproj
-a---          2024/06/19    11:03           1061 llm_onnx_test.sln
-a---          2024/06/19    12:43           3089 llm_server.cs
-a---          2024/06/19    12:43            198 llm_server.tscn
-a---          2024/06/19    11:34            597 project.godot

dotnet add package Microsoft.ML.OnnxRuntimeGenAI.DirectML

<Project Sdk="Godot.NET.Sdk/4.2.2">
  <PropertyGroup>
    <TargetFramework>net6.0</TargetFramework>
    <TargetFramework Condition=" '$(GodotTargetPlatform)' == 'android' ">net7.0</TargetFramework>
    <TargetFramework Condition=" '$(GodotTargetPlatform)' == 'ios' ">net8.0</TargetFramework>
    <EnableDynamicLoading>true</EnableDynamicLoading>
  </PropertyGroup>
  <ItemGroup>
    <PackageReference Include="Microsoft.ML.OnnxRuntimeGenAI.DirectML" Version="0.2.0" />
  </ItemGroup>
</Project>

pip install -U "huggingface_hub[cli]"

huggingface-cli download microsoft/Phi-3-mini-4k-instruct-onnx --include directml/* --local-dir .

using Godot;
using System;
using System.IO;
using System.Threading;
using System.Threading.Tasks;
using System.Collections.Generic;
using System.Diagnostics;
using System.Text;
using Microsoft.ML.OnnxRuntimeGenAI;

public partial class llm_server : Node
{
	private Model? model = null;
	private Tokenizer? tokenizer = null;
	private readonly string ModelDir = Path.Combine(@"E:\godot_projects\llm_onnx_test\Models\directml\directml-int4-awq-block-128");
	public string input_text = "";
	public List<string> AntiPrompts = new List<string> {"<|end|>", "<|user|>", "<|assistant|>", "\n"};
	[Signal]
	public delegate void generateStartEventHandler();
	[Signal]
	public delegate void generatingEventHandler(string text);
	[Signal]
	public delegate void generateEndEventHandler();
	public override void _Ready()
	{
		generateStart += generate;
	}
	
	public void InitializeModel()
	{
		GD.Print("Loading model...");
		var sw = Stopwatch.StartNew();
		model = new Model(ModelDir);
		tokenizer = new Tokenizer(model);
		sw.Stop();
		GD.Print($"Model loading took {sw.ElapsedMilliseconds} ms");
	}
	public void generate(){
		Thread thread = new Thread(new ThreadStart(run_llm));
		thread.Start();
	}
	public async void run_llm(){
		string output_text = "";
		if(model != null)
		{
			var systemPrompt = "あなたは優秀なAIアシスタントです";
			var prompt = $@"<|system|>{systemPrompt}<|end|>
			<|user|>{this.input_text}<|end|>
			<|assistant|>";
			
			await foreach (var part in InferStreaming(prompt))
			{
				output_text += part;
				CallDeferred("emit_signal", SignalName.generating, output_text);
			}
		}
		CallDeferred("emit_signal", SignalName.generateEnd);
	}
	public async IAsyncEnumerable<string> InferStreaming(string prompt)
	{
		if (model == null || tokenizer == null)
		{
			throw new InvalidOperationException("Model is not ready");
		}

		var generatorParams = new GeneratorParams(model);

		var sequences = tokenizer.Encode(prompt);

		generatorParams.SetSearchOption("max_length", 2048);
		generatorParams.SetSearchOption("do_sample", true);
		
		generatorParams.SetSearchOption("temperature", 0.8);
		generatorParams.SetSearchOption("top_k", 50);
		generatorParams.SetSearchOption("top_p", 0.95);
		generatorParams.SetInputSequences(sequences);
		generatorParams.TryGraphCaptureWithMaxBatchSize(1);

		using var tokenizerStream = tokenizer.CreateStream();
		using var generator = new Generator(model, generatorParams);
		StringBuilder stringBuilder = new();
		while (!generator.IsDone())
		{
			string part;
			try
			{
				await Task.Delay(10).ConfigureAwait(false);
				generator.ComputeLogits();
				generator.GenerateNextToken();
				part = tokenizerStream.Decode(generator.GetSequence(0)[^1]);
				stringBuilder.Append(part);
				bool break_flg = false;
				foreach (string token in this.AntiPrompts)
				{
					if (stringBuilder.ToString().Contains(token))
					{
						break_flg = true;
						break;
					}
				}
				if (break_flg){
					break;
				}
			}
			catch (Exception ex)
			{
				Debug.WriteLine(ex);
				break;
			}
			yield return part;
		}
	}
}

generatorParams.SetSearchOption("do_sample", true);
generatorParams.SetSearchOption("temperature", 0.8);
generatorParams.SetSearchOption("top_k", 50);
generatorParams.SetSearchOption("top_p", 0.95);

extends Node2D

var server_class := preload("res://llm_server.tscn")
var server

func _ready():
	server = server_class.instantiate()
	server.generating.connect(_on_llm_generate_label_generating)
	server.generateEnd.connect(_on_llm_generate_label_generate_end)
	server.InitializeModel()
	add_child(server)

func _on_button_pressed():
	var input_text :String = $Panel/VBoxContainer/TextEdit.text
	$Panel/VBoxContainer/ScrollContainer/Label.text = ""
	server.input_text = input_text
	server.emit_signal('generateStart')
	$Panel/VBoxContainer/Button.disabled = true
	
func _on_llm_generate_label_generating(text):
	$Panel/VBoxContainer/ScrollContainer/Label.text = text

func _on_llm_generate_label_generate_end():
	pass
	$Panel/VBoxContainer/Button.disabled = false