Internal Test: Removes the core alphanumeric token to verify that the tokenizer falls back to character-by-character tokenization.

ID: 1047

See: Remove 
				
					using uCalcSoftware;

var uc = new uCalc();
var t = new uCalc.Transformer();
string text = "word game";
t.FromTo("{token:1}", "[{@Self}]");

Console.WriteLine("--- Before Remove ---");
// By default, 'word' is a single alphanumeric token.
Console.WriteLine(t.Transform(text));

// Remove the alphanumeric token definition.
var alphaToken = t.Tokens.ByName("_token_alphanumeric");
t.Tokens.Remove(alphaToken);

// The transformer must be reset with the original text for the change to apply.
t.Text = text;

Console.WriteLine("");
Console.WriteLine("--- After Remove ---");
// Now, 'word' is no longer a single token. The fallback '.' token
// matches each character individually.
Console.WriteLine(t.Transform());
				
			
--- Before Remove ---
[word] [game]

--- After Remove ---
[w][o][r][d] [g][a][m][e]
Explore 600+ other examples
				
					#include <iostream>
#include "uCalc.h"

using namespace std;
using namespace uCalcSoftware;

int main() {
   uCalc uc;
   uCalc::Transformer t;
   string text = "word game";
   t.FromTo("{token:1}", "[{@Self}]");

   cout << "--- Before Remove ---" << endl;
   // By default, 'word' is a single alphanumeric token.
   cout << t.Transform(text) << endl;

   // Remove the alphanumeric token definition.
   auto alphaToken = t.Tokens().ByName("_token_alphanumeric");
   t.Tokens().Remove(alphaToken);

   // The transformer must be reset with the original text for the change to apply.
   t.Text(text);

   cout << "" << endl;
   cout << "--- After Remove ---" << endl;
   // Now, 'word' is no longer a single token. The fallback '.' token
   // matches each character individually.
   cout << t.Transform() << endl;
}
				
			
--- Before Remove ---
[word] [game]

--- After Remove ---
[w][o][r][d] [g][a][m][e]
Explore 600+ other examples
				
					Imports System
Imports uCalcSoftware
Public Module Program
   Public Sub Main()
      Dim uc As New uCalc()
      Dim t As New uCalc.Transformer()
      Dim text As String = "word game"
      t.FromTo("{token:1}", "[{@Self}]")
      
      Console.WriteLine("--- Before Remove ---")
      '// By default, 'word' is a single alphanumeric token.
      Console.WriteLine(t.Transform(text))
      
      '// Remove the alphanumeric token definition.
      Dim alphaToken = t.Tokens.ByName("_token_alphanumeric")
      t.Tokens.Remove(alphaToken)
      
      '// The transformer must be reset with the original text for the change to apply.
      t.Text = text
      
      Console.WriteLine("")
      Console.WriteLine("--- After Remove ---")
      '// Now, 'word' is no longer a single token. The fallback '.' token
      '// matches each character individually.
      Console.WriteLine(t.Transform())
   End Sub
End Module
				
			
--- Before Remove ---
[word] [game]

--- After Remove ---
[w][o][r][d] [g][a][m][e]
Explore 600+ other examples