AleaTK throw new NotImplementedException() on new project

by Rusty Nail   Last Updated August 14, 2019 04:26 AM

I am testing AleaTK for a task I need done.

The GitHub repo is Here.

If I download the Repo and test the PTB example, it works fine!

If I use Nuget and download the AleaTK Nuget and add the Classes: 'Lstm.cs', 'LstmRnnType.cs' with all the extensions necessary, I get an error:

The method or operation is not implemented.

Now, the call is coming from:

Optimizer.Forward();

and Optimizer is:

Optimizer = new GradientDescentOptimizer(ctx, Loss.Loss, cfg.LearningRate, new GlobalNormGradientClipper(cfg.MaxGradNorm));

The same code as in the PTB project.

The Forward method, is overridden as far as I can tell:

public override void Forward(Executor executor)

and

public override void Forward(Executor executor)

In both classes. Lstm and Rnn.

In looking through the code, there is a LOT of throw new NotImplementedException() with zero indication of where the problem is coming from.

I wonder if there is a quick way I can find exactly where this error is coming from and why, what exactly I have missed.

at System.Threading.Tasks.Task.ThrowIfExceptional(Boolean includeTaskCanceledExceptions) at System.Threading.Tasks.Task.Wait(Int32 millisecondsTimeout, CancellationToken cancellationToken) at AleaTK.Assignment.Run()
at AleaTK.ML.Executor.AssignTensor[T](Variable1 variable, Expr1 expr) at AleaTK.ML.Executor.Forward() at AleaConsole.Model..ctor(Context ctx, Config cfg, Boolean isTraining, Boolean usingCuDnn) in D:\C# Projects\Machine Learning Examples\Alea TK for ML\AleaConsole\AleaConsole\Program.cs:line 693 at AleaConsole.Program.Main(String[] args) in D:\C# Projects\Machine Learning Examples\Alea TK for ML\AleaConsole\AleaConsole\Program.cs:line 116

namespace AleaConsole
{



#region Using Statements:



using System;
using System.IO;
using System.Net;
using System.Linq;
using System.Text;
using System.Diagnostics;
using System.IO.Compression;
using System.Collections.Generic;

using Alea;
using AleaTK;
using AleaTK.ML;
using AleaTK.ML.Operator;
using static AleaTK.Library;
using static AleaTK.ML.Library;
using Context = AleaTK.Context;

using NUnit.Framework;
using ICSharpCode.SharpZipLib.Tar;



#endregion


public static class Extensions
{



public static void Iter<T>(this IEnumerable<T> ie, Action<T, int> action)
{

var i = 0;

foreach (var e in ie)
{
action(e, i++);
}
}
}



class Program
{



#region Fields:



public const int TestHiddenSize = -1;



public const bool Profiling = false;



public const int TestMaxMaxEpoch = Profiling ? 1 : -1;



public const string DataPath = @"Data\PTB\simple-examples\data";



#endregion



#region Properties:



#endregion




static void Main(string[] args)
{


bool isConsole = true;
bool usingCuDnn = true;


Console.WriteLine($"UsingCUDNN({usingCuDnn}), Config: small");

var ptb = new Data(DataPath);
var ctx = Context.GpuContext(0);

Config cfg, cfgValid, cfgTest, cfgInteractive;
cfg = Config.Small(batchSize: 20);
cfgValid = Config.Small(batchSize: 20);
cfgTest = Config.Small(batchSize: 1, numSteps: 1);
cfgInteractive = Config.Small(batchSize: 1, numSteps: 10);

Assert.AreEqual(ptb.WordToIdDict.Count, cfg.VocabSize);
Assert.AreEqual(ptb.WordToIdDict.Count, cfgValid.VocabSize);
Assert.AreEqual(ptb.WordToIdDict.Count, cfgTest.VocabSize);
Assert.AreEqual(ptb.WordToIdDict.Count, cfgInteractive.VocabSize);

var model = new Model(ctx, cfg, isTraining: true, usingCuDnn: usingCuDnn);
var modelValid = new Model(ctx, cfgValid, isTraining: false, usingCuDnn: usingCuDnn);
var modelTest = new Model(ctx, cfgTest, isTraining: false, usingCuDnn: usingCuDnn);
var modelInteractive = new Model(ctx, cfgInteractive, isTraining: false, usingCuDnn: usingCuDnn);

for (var i = 0; i < cfg.MaxMaxEpoch; ++i)
{
var lrDecay = Math.Pow(cfg.LrDecay, Math.Max(i - cfg.MaxEpoch, 0.0));
var learningRate = cfg.LearningRate * lrDecay;

Console.WriteLine($"Epoch: {i + 1} Learning rate: {learningRate:F3}");
var trainPerplexity = model.RunEpoch(ptb.TrainData, learningRate: learningRate, verbose: true);
Console.WriteLine($"Epoch: {i + 1} Train Perplexity: {trainPerplexity:F3}");

if (!Profiling)
{
modelValid.CopyWeightsFrom(model);
var validPerplexity = modelValid.RunEpoch(ptb.ValidData);
Console.WriteLine($"Epoch: {i + 1} Valid Perplexity: {validPerplexity:F3}");
}
}

if (!Profiling)
{
modelTest.CopyWeightsFrom(model);
Console.WriteLine("Testing with test data, this is slow, since batch size is set to small...");
var testPerplexity = modelTest.RunEpoch(ptb.TestData, verbose: true);
Console.WriteLine($"Test Perplexity: {testPerplexity:F3}");
}

if (!Profiling && isConsole)
{
var inputs = new int[cfgInteractive.NumSteps, 1];
modelInteractive.CopyWeightsFrom(model);

// since the entropy and softmax are merged, so we have to allocate the target (label) tensor
modelInteractive.Optimizer.AssignTensor(modelInteractive.Targets, inputs.AsTensor());

while (true)
{
Console.WriteLine();
Console.WriteLine($"Enter some words (less than {cfgInteractive.NumSteps} words)");
var readLine = Console.ReadLine();
if (readLine == null) break;
var line = readLine.Trim(' ', '\t', '\r', '\n');
var words = line.Split(new[] { ' ', '\t', '\r', '\n' }, StringSplitOptions.RemoveEmptyEntries);
if (words.Length <= 0 || words.Length > cfgInteractive.NumSteps) continue;

for (var i = 0; i < cfgInteractive.NumSteps; ++i)
{
if (i < words.Length)
{
inputs[i, 0] = ptb.WordToId(words[i]);
}
else
{
inputs[i, 0] = ptb.WordToId("<unk>");
}
}

Console.WriteLine("Your inputs are:");
for (var i = 0; i < cfgInteractive.NumSteps; ++i)
{
Console.Write($"{ptb.IdToWord(inputs[i, 0])} ");
}
Console.WriteLine();

modelInteractive.ResetStates();
modelInteractive.Optimizer.AssignTensor(modelInteractive.Inputs, inputs.AsTensor());
modelInteractive.Optimizer.Forward();

var logPred = modelInteractive.Optimizer.GetTensor(modelInteractive.Loss.LogPred).ToArray2D();
var pred = new List<IndexAndProb>();
var totalProb = 0.0;
for (var i = 0; i < cfgInteractive.VocabSize; ++i)
{
var p = new IndexAndProb { Index = i, Prob = Math.Exp(logPred[words.Length - 1, i]) };
pred.Add(p);
totalProb += p.Prob;
}
Console.WriteLine($"Total probability: {totalProb:F4}");
pred.Sort();
Console.WriteLine("Candidates are:");
pred.Take(10).Iter((x, o) => { Console.WriteLine($" {x.Prob:P2} --> {ptb.IdToWord(x.Index)}"); });
}
}
}
}



public class Data
{


private static void Decompress(string src, string dst)
{
using (var originalFileStream = File.OpenRead(src))
using (var decompressedFileStream = File.Create(dst))
using (var decompressionStream = new GZipStream(originalFileStream, CompressionMode.Decompress))
{
decompressionStream.CopyTo(decompressedFileStream);
}
}



public static void EnsureDataFile()
{
const string doneFileName = @"Data\PTB.done";
const string url = @"http://www.fit.vutbr.cz/~imikolov/rnnlm/simple-examples.tgz";

if (!Directory.Exists("Data"))
{
Directory.CreateDirectory("Data");
}

if (!File.Exists(doneFileName))
{
using (var client = new WebClient())
{
Console.WriteLine($"Downloading {url} ...");
client.DownloadFile(url, @"Data\PTB.tgz");
}

Decompress(@"Data\PTB.tgz", @"Data\PTB.tar");

using (var tarFile = File.OpenRead(@"Data\PTB.tar"))
using (var tarArchive = TarArchive.CreateInputTarArchive(tarFile))
{
tarArchive.ExtractContents(@"Data\PTB");
}

using (var doneFile = File.CreateText(doneFileName))
{
doneFile.WriteLine($"{DateTime.Now}");
}
}
}



public static List<string> ReadWords(string path)
{

var totalWords = new List<string>();
using (var file = File.Open(path, FileMode.Open))
using (var reader = new StreamReader(file, Encoding.UTF8))
{
while (!reader.EndOfStream)
{
var line = reader.ReadLine();
var words = line?.Split(new[] { ' ' }, StringSplitOptions.RemoveEmptyEntries);
if (!(words?.Length > 0)) continue;
totalWords.AddRange(words);
totalWords.Add("<eos>");
}
}
return totalWords;
}



public static void BuildVocabulary(string path, out Dictionary<string, int> word2id, out Dictionary<int, string> id2word)
{
var data = ReadWords(path).Distinct().ToList();
data.Sort();
word2id = new Dictionary<string, int>();
id2word = new Dictionary<int, string>();
var id = 0;
foreach (var word in data)
{
word2id.Add(word, id);
id2word.Add(id, word);
id++;
}
}



public readonly Dictionary<string, int> WordToIdDict;



public readonly Dictionary<int, string> IdToWordDict;



public readonly int[] TrainData;



public readonly int[] ValidData;



public readonly int[] TestData;



public int WordToId(string word)
{
return WordToIdDict.ContainsKey(word) ? WordToIdDict[word] : WordToIdDict["<unk>"];
}



public string IdToWord(int id)
{
return IdToWordDict[id];
}



public Data(string dataPath)
{

EnsureDataFile();

var trainPath = Path.Combine(dataPath, "ptb.train.txt");
var validPath = Path.Combine(dataPath, "ptb.valid.txt");
var testPath = Path.Combine(dataPath, "ptb.test.txt");

BuildVocabulary(trainPath, out WordToIdDict, out IdToWordDict);

TrainData = ReadWords(trainPath).Select(WordToId).ToArray();
ValidData = ReadWords(validPath).Select(WordToId).ToArray();
TestData = ReadWords(testPath).Select(WordToId).ToArray();
}



public List<string> GetWords(int from, int to)
{

var words = new List<string>();

for (var i = from; i < to; ++i)
words.Add(IdToWordDict[TrainData[i]]);

return words;
}



public class Batch
{
public int[,] Inputs { get; set; }
public int[,] Targets { get; set; }
}



public static IEnumerable<Batch> Iterator(int[] rawData, int numSteps, int batchSize)
{

var dataLen = rawData.Length;
var batchLen = dataLen / batchSize;
var data = new int[batchSize, batchLen];
for (var i = 0; i < batchSize; ++i)
{
for (var j = 0; j < batchLen; ++j)
{
data[i, j] = rawData[batchLen * i + j];
}
}

var epochSize = (batchLen - 1) / numSteps;

Util.EnsureTrue(epochSize != 0);

for (var i = 0; i < epochSize; ++i)
{
var x = new int[numSteps, batchSize];
var y = new int[numSteps, batchSize];

for (var t = 0; t < numSteps; ++t)
{
for (var j = 0; j < batchSize; ++j)
{
x[t, j] = data[j, numSteps * i + t];
y[t, j] = data[j, numSteps * i + t + 1];
}
}

yield return new Batch { Inputs = x, Targets = y };
}
}
}



public class IndexAndProb : IComparable
{


public int Index;



public double Prob;



public int CompareTo(object obj)
{
var o = (IndexAndProb)obj;
if (Prob == o.Prob) return 0;
return Prob > o.Prob ? -1 : 1;
}


public override string ToString()
{
return $"({Index}:{Prob:F2})";
}
}



public class Config
{

public double InitScale;
public double LearningRate;
public double MaxGradNorm;
public int NumLayers;
public int NumSteps;
public int HiddenSize;
public int MaxEpoch;    // learning rate start to reduce after this epoch
public int MaxMaxEpoch; // epoches to run
public double KeepProb;
public double LrDecay;
public int BatchSize;
public int VocabSize;

public const bool Profiling = false;

public const int TestMaxMaxEpoch = Profiling ? 1 : -1;

public const int TestHiddenSize = -1;

public static Config Small(int batchSize = 20, int numSteps = 20, double keepProb = 1.0)
{

return new Config
{

InitScale = 0.1,
LearningRate = 1.0,
MaxGradNorm = 5.0,
NumLayers = 2,
NumSteps = numSteps,
HiddenSize = TestHiddenSize > 0 ? TestHiddenSize : 200,
MaxEpoch = 4,
MaxMaxEpoch = TestMaxMaxEpoch > 0 ? TestMaxMaxEpoch : 13,
KeepProb = keepProb,
LrDecay = 0.5,
BatchSize = batchSize,
VocabSize = 10000
};
}



public static Config Medium(int batchSize = 20, int numSteps = 35, double keepProb = 0.5)
{

return new Config
{

InitScale = 0.05,
LearningRate = 1.0,
MaxGradNorm = 5.0,
NumLayers = 2,
NumSteps = numSteps,
HiddenSize = TestHiddenSize > 0 ? TestHiddenSize : 650,
MaxEpoch = 6,
MaxMaxEpoch = TestMaxMaxEpoch > 0 ? TestMaxMaxEpoch : 39,
KeepProb = keepProb,
LrDecay = 0.8,
BatchSize = batchSize,
VocabSize = 10000
};
}



public static Config Large(int batchSize = 20, int numSteps = 35, double keepProb = 0.35)
{

return new Config
{

InitScale = 0.04,
LearningRate = 1.0,
MaxGradNorm = 10.0,
NumLayers = 2,
NumSteps = numSteps,
HiddenSize = TestHiddenSize > 0 ? TestHiddenSize : 1500,
MaxEpoch = 14,
MaxMaxEpoch = TestMaxMaxEpoch > 0 ? TestMaxMaxEpoch : 55,
KeepProb = keepProb,
LrDecay = 1.0 / 1.15,
BatchSize = batchSize,
VocabSize = 10000
};
}
}



public class Model
{



#region Fields:



public enum ConfigType
{

Small = 0,
Medium,
Large
}



public const string DataPath = @"Data\PTB\simple-examples\data";



public const bool Profiling = false;



public const int TestMaxMaxEpoch = Profiling ? 1 : -1;



public const int TestHiddenSize = -1;



public const ConfigType CfgType = ConfigType.Small;  // ConfigType.Small, ConfigType.Large



#endregion



#region Properties:



public Config Config { get; }



public bool IsTraining { get; }



public bool UsingCuDnn { get; }



public Variable<int> Inputs { get; }



public Variable<int> Targets { get; }



public Embedding<float> Embedding { get; }



public Variable<float> EmbeddedOutput { get; }



public Lstm<float>[] RnnDirect { get; }



public Rnn<float> RnnAccelerated { get; }



public Variable<float> RnnOutput { get; }



public FullyConnected<float> FC { get; }



public SoftmaxCrossEntropySparse<float> Loss { get; }



public GradientDescentOptimizer Optimizer { get; }



#endregion



public Model(Context ctx, Config cfg, bool isTraining = true, bool usingCuDnn = true)
{

Config = cfg;
IsTraining = isTraining;
UsingCuDnn = usingCuDnn;

Inputs = Variable<int>(PartialShape.Create(cfg.NumSteps, cfg.BatchSize));
Targets = Variable<int>(PartialShape.Create(cfg.NumSteps, cfg.BatchSize));

// embedding
Embedding = new Embedding<float>(Inputs, cfg.VocabSize, cfg.HiddenSize, initScale: cfg.InitScale);

// add dropout
EmbeddedOutput = Embedding.Output;
if (isTraining && cfg.KeepProb < 1.0)
{
var dropout = new Dropout<float>(EmbeddedOutput, dropoutProb: 1.0 - cfg.KeepProb);
EmbeddedOutput = dropout.Output;
}

// rnn layer, dropout for intermediate lstm layers and for output
if (usingCuDnn)
{
RnnAccelerated = new Rnn<float>(new LstmRnnType(forgetBiasInit: 0.0), EmbeddedOutput, cfg.NumLayers, cfg.HiddenSize, isTraining: isTraining, dropout: isTraining && cfg.KeepProb < 1.0 ? 1.0 - Config.KeepProb : 0.0);
RnnOutput = RnnAccelerated.Y;
if (isTraining && cfg.KeepProb < 1.0)
{
var dropout = new Dropout<float>(RnnOutput, dropoutProb: 1.0 - cfg.KeepProb);
RnnOutput = dropout.Output;
}
}
else
{
RnnDirect = new Lstm<float>[cfg.NumLayers];
for (var i = 0; i < cfg.NumLayers; ++i)
{
var lstm = new Lstm<float>(i == 0 ? EmbeddedOutput : RnnOutput, cfg.HiddenSize, forgetBiasInit: 0.0);
RnnDirect[i] = lstm;
RnnOutput = lstm.Y;
if (isTraining && cfg.KeepProb < 1.0)
{
var dropout = new Dropout<float>(RnnOutput, dropoutProb: 1.0 - cfg.KeepProb);
RnnOutput = dropout.Output;
}
}
}

FC = new FullyConnected<float>(RnnOutput.Reshape(RnnOutput.Shape[0] * RnnOutput.Shape[1], RnnOutput.Shape[2]), cfg.VocabSize);

Loss = new SoftmaxCrossEntropySparse<float>(FC.Output, Targets.Reshape(Targets.Shape[0] * Targets.Shape[1]));

Optimizer = new GradientDescentOptimizer(ctx, Loss.Loss, cfg.LearningRate, new GlobalNormGradientClipper(cfg.MaxGradNorm));

// warmup to force JIT compilation to get timings without JIT overhead
Optimizer.Initalize();

ResetStates();

Optimizer.AssignTensor(Inputs, Fill(Shape.Create(Inputs.Shape.AsArray), 0));
Optimizer.AssignTensor(Targets, Fill(Shape.Create(Targets.Shape.AsArray), 0));

Optimizer.Forward();

if (isTraining)
{
Optimizer.Backward();
}

// now reset states
Optimizer.Initalize();
ResetStates();
}



public void CopyWeightsFrom(Model o)
{

Optimizer.AssignTensor(Embedding.Weights, o.Optimizer.GetTensor(o.Embedding.Weights));
Optimizer.AssignTensor(FC.Weights, o.Optimizer.GetTensor(o.FC.Weights));
Optimizer.AssignTensor(FC.Bias, o.Optimizer.GetTensor(o.FC.Bias));
if (UsingCuDnn)
{
Util.EnsureTrue(o.UsingCuDnn);
Optimizer.AssignTensor(RnnAccelerated.W, o.Optimizer.GetTensor(o.RnnAccelerated.W));
}
else
{
Util.EnsureTrue(!o.UsingCuDnn);
for (var i = 0; i < Config.NumLayers; ++i)
{
Optimizer.AssignTensor(RnnDirect[i].W, o.Optimizer.GetTensor(o.RnnDirect[i].W));
}
}
}



public void ResetStates()
{

if (UsingCuDnn)
{
Optimizer.AssignTensor(RnnAccelerated.CX, Fill(Shape.Create(RnnAccelerated.CX.Shape.AsArray), 0.0f));
Optimizer.AssignTensor(RnnAccelerated.HX, Fill(Shape.Create(RnnAccelerated.HX.Shape.AsArray), 0.0f));
}
else
{

for (var i = 0; i < Config.NumLayers; ++i)
{
var lstm = RnnDirect[i];
var shape = Shape.Create(Config.BatchSize, lstm.HiddenSize);
Optimizer.AssignTensor(lstm.CX, Fill(shape, 0.0f));
Optimizer.AssignTensor(lstm.HX, Fill(shape, 0.0f));
}
}
}



public void CopyStates()
{

if (UsingCuDnn)
{
Optimizer.AssignTensor(RnnAccelerated.CX, Optimizer.GetTensor(RnnAccelerated.CY));
Optimizer.AssignTensor(RnnAccelerated.HX, Optimizer.GetTensor(RnnAccelerated.HY));
}
else
{
for (var i = 0; i < Config.NumLayers; ++i)
{
var lstm = RnnDirect[i];
Optimizer.AssignTensor(lstm.CX, Optimizer.GetTensor(lstm.CY));
Optimizer.AssignTensor(lstm.HX, Optimizer.GetTensor(lstm.HY));
}
}
}



public double RunEpoch(int[] data, double learningRate = 1.0, bool verbose = false)
{

var cfg = Config;
var isTraining = IsTraining;
var epochSize = (data.Length / cfg.BatchSize - 1) / cfg.NumSteps;
var time = Stopwatch.StartNew();
var costs = 0.0;
var iters = 0;
var step = 0;
var firstBatch = true;

foreach (var batch in Data.Iterator(data, cfg.NumSteps, cfg.BatchSize))
{
Optimizer.AssignTensor(Inputs, batch.Inputs.AsTensor());
Optimizer.AssignTensor(Targets, batch.Targets.AsTensor());

if (firstBatch)
{
ResetStates();
firstBatch = false;
}
else
{
CopyStates();
}

Optimizer.Forward();

if (isTraining)
{
Optimizer.Backward();
Optimizer.Optimize(learningRate);
}

var loss = Optimizer.GetTensor(Loss.Loss).ToScalar();
var cost = loss / cfg.BatchSize;
costs += cost;
iters += cfg.NumSteps;

if (Profiling || (verbose && (step % (epochSize / 10) == 10)))
{
var perplexity = Math.Exp(costs / iters);
var wps = (iters * cfg.BatchSize) / (time.Elapsed.TotalMilliseconds / 1000.0);

Console.WriteLine($"{step:D4}: {step * 1.0 / epochSize:F3} perplexity: {perplexity:F3} speed:{wps:F0} wps cost: {cost:F3}");
}

if (Profiling && step > 5) break;

step++;
}

return Math.Exp(costs / iters);
}
}
}

The LSTM Class: Here

<?xml version="1.0" encoding="utf-8"?>
<configuration>
<configSections>
<section name="aleaSettings" type="Alea.Settings, Alea" />
</configSections>
<aleaSettings>
<cuBLAS version="9.0" />
<cuRAND version="9.0" />
</aleaSettings>
<startup>
<supportedRuntime version="v4.0" sku=".NETFramework,Version=v4.5.2" />
</startup>
<runtime>
<assemblyBinding xmlns="urn:schemas-microsoft-com:asm.v1">
  <dependentAssembly>
    <assemblyIdentity name="FSharp.Core" publicKeyToken="b03f5f7f11d50a3a" culture="neutral" />
    <bindingRedirect oldVersion="0.0.0.0-4.7.0.0" newVersion="4.7.0.0" />
  </dependentAssembly>
</assemblyBinding>
</runtime>
</configuration>
Tags : c# aleatk


Related Questions