From aabf7eb566d75804e51f0dc111a0b6f62c75f6fb Mon Sep 17 00:00:00 2001 From: TreetopFlyer Date: Mon, 26 Jul 2021 09:27:38 -0400 Subject: [PATCH 01/13] all methods passing tests --- m.js | 21 +++++++++------------ m.test.js | 39 +++++++++++++++++++++++++++++++++------ methods.md | 6 +++--- 3 files changed, 45 insertions(+), 21 deletions(-) diff --git a/m.js b/m.js index e2914fb..1a85dac 100644 --- a/m.js +++ b/m.js @@ -2,7 +2,7 @@ const M = { Iterate: { - New(inDimensions, inCount, inFunction) + Loop: (inDimensions, inCount, inFunction)=> { let row, i, outputCloud, outputVector; outputCloud = []; @@ -17,17 +17,14 @@ const M = } return outputCloud; }, - Old(inCloud, inFunction) - { - return M.Iterate.New(inCloud[0].length, inCloud.length, inFunction); - } + Edit: (inCloud, inFunction)=> inCloud.map(row=>row.map(inFunction)) }, Create: { - Box: (inV1, inV2, inCount)=> M.Iterate.New(inV1.length, inCount, (i, row)=> inV1[i]+(inV2[i]-inV1[i])*Math.random()), - Transpose: (inCloud)=> M.Iterate.New(inCloud.length, inCloud[0].length, (i, row)=> inCloud[i][row]), - Outer: (inV1, inV2)=> M.Iterate.New(inV1.length, inV2.length, (i, row)=> inV1[i]*inV2[row]), - Clone: (inCloud)=> M.Iterate.Old(inCloud, (i, row)=> inCloud[row][i]) + Box: (inV1, inV2, inCount)=> M.Iterate.Loop(inV1.length, inCount, (i, row)=> inV1[i]+(inV2[i]-inV1[i])*Math.random()), + Transpose: (inCloud)=> M.Iterate.Loop(inCloud.length, inCloud[0].length, (i, row)=> inCloud[i][row]), + Outer: (inV1, inV2)=> M.Iterate.Loop(inV1.length, inV2.length, (i, row)=> inV1[i]*inV2[row]), + Clone: (inCloud)=> M.Iterate.Edit(inCloud, i=> i) }, Mutate: { @@ -45,9 +42,9 @@ const M = Subtract: (inCloud1, inCloud2)=> inCloud1.map((row, rowIndex)=> M.Single.Subtract(row, inCloud2[rowIndex])), Multiply: (inCloud1, inCloud2)=> inCloud1.map((row, rowIndex)=> M.Single.Multiply(row, inCloud2[rowIndex])), Affine: (inCloud, inMatrix)=> inCloud.map(row=> M.Single.Affine(row, inMatrix)), - Sigmoid: (inCloud)=> M.Iterate.Old(inCloud, i=>1/(1+Math.Pow(Math.E, i))), - Derivative: (inCloud)=> M.Iterate.Old(inCloud, i=>i*(1-i)), - Scale: (inCloud, inScalar)=> M.Iterate.Old(inCloud, i=>i*inScalar) + Sigmoid: (inCloud)=> M.Iterate.Edit(inCloud, i=>1/(1+Math.pow(Math.E, -i))), + Derivative: (inCloud)=> M.Iterate.Edit(inCloud, i=>i*(1-i)), + Scale: (inCloud, inScalar)=> M.Iterate.Edit(inCloud, i=>i*inScalar) } } diff --git a/m.test.js b/m.test.js index 823298d..2bbab28 100644 --- a/m.test.js +++ b/m.test.js @@ -1,16 +1,25 @@ import { assert, assertEquals } from "https://deno.land/std@0.102.0/testing/asserts.ts"; import { default as M } from "./m.js"; -Deno.test("Iterate.New", ()=> +Deno.test("Iterate.Loop", ()=> { let dimensions = 3; let count = 4; - let cloud = M.Iterate.New(dimensions, count, (i, j)=>i+j); + let cloud = M.Iterate.Loop(dimensions, count, (i, j)=>i+j); assertEquals(cloud.length, count, "correct count"); assertEquals(cloud[0].length, dimensions, "correct dimensions"); assertEquals(cloud[0][0], 0); assertEquals(cloud[3][2], 5, "correct output"); }); +Deno.test("Iterate.Edit", ()=> +{ + let c = [[1, 2], [3, 4]] + let t = M.Iterate.Edit(c, (i)=>i); + assertEquals(t.length, c.length, "correct count"); + assertEquals(t[0][0], c[0][0], "correct dimensions"); + assertEquals(t[1][1], c[1][1], "correct placement"); +}); + Deno.test("Create.Box", ()=> { @@ -30,7 +39,6 @@ Deno.test("Create.Box", ()=> } } }); - Deno.test("Create.Transpose", ()=> { let v1 = [1, 2, 3]; @@ -41,7 +49,6 @@ Deno.test("Create.Transpose", ()=> assertEquals(tpose[0][0], v1[0]); assertEquals(tpose[0][1], v2[0], "correct placement"); }); - Deno.test("Create.Outer", ()=> { let v1 = [1, 2, 3]; @@ -51,7 +58,6 @@ Deno.test("Create.Outer", ()=> assertEquals(outer[0].length, v1.length, "correct dimensions"); assertEquals(outer[1][0], v1[0]*v2[1], "correct placement") }); - Deno.test("Create.Clone", ()=> { let v1 = [1, 2, 3]; @@ -62,6 +68,7 @@ Deno.test("Create.Clone", ()=> assertEquals(clone[1][0], v2[0], "correct placement"); }); + Deno.test("Mutate.Pad", ()=> { let matrix = [ @@ -133,8 +140,28 @@ Deno.test("Batch.Scale", ()=> let c = [[1, 2], [3, 4]]; let s = 0.5; let t = M.Batch.Scale(c, s); + console.log(t); assertEquals(t.length, 2, "correct count"); assertEquals(t[0].length, 2, "correct dimensions"); - console.log(t); assertEquals(t[1][0], 1.5, "correct placement"); }); + + +Deno.test("Batch.Sigmoid", ()=> +{ + let m = [[-1000, 1000]]; + let t = M.Batch.Sigmoid(m); + console.log(t); + assertEquals(t.length, 1, "correct count"); + assertEquals(t[0].length, 2, "correct dimensions"); + assert(t[0][0]>=0 && t[0][0]<0.5); + assert(t[0][1]<=1 && t[0][1]>0.5, "correct placement"); +}); +Deno.test("Batch.Derivative", ()=> +{ + let m = [[-1000, 0, 1000]]; + let t = M.Batch.Derivative(M.Batch.Sigmoid(m)); + assertEquals(t.length, 1, "correct count"); + assertEquals(t[0].length, 3, "correct dimensions"); + assert(t[0][0]t[0][2]); +}); \ No newline at end of file diff --git a/methods.md b/methods.md index b149d13..e365088 100644 --- a/methods.md +++ b/methods.md @@ -7,8 +7,8 @@ pad(inCloud) // done unpad(inCloud) // done transform(inCloud, inMatrix) // done -sigmoid(inCloud) // 1/(1+e^x) // -derivative(inCloud) // x*(1-x) // -scale(inCloud1, inV) // +sigmoid(inCloud) // 1/(1+e^x) // done +derivative(inCloud) // x*(1-x) // done +scale(inCloud1, inV) // done subtract(inCloud1, inCloud2) // done multiply(inCloud1, inCloud2) // done \ No newline at end of file From 194c66aa36bfe640ca180982498f29f2999eee23 Mon Sep 17 00:00:00 2001 From: TreetopFlyer Date: Mon, 26 Jul 2021 09:46:37 -0400 Subject: [PATCH 02/13] lint checks --- m.js | 12 ++++---- m.test.js | 90 ++++++++++++++++++++++++------------------------------- 2 files changed, 45 insertions(+), 57 deletions(-) diff --git a/m.js b/m.js index 1a85dac..37a74e5 100644 --- a/m.js +++ b/m.js @@ -4,14 +4,14 @@ const M = { Loop: (inDimensions, inCount, inFunction)=> { - let row, i, outputCloud, outputVector; - outputCloud = []; - for(row=0; row M.Iterate.Loop(inV1.length, inCount, (i, row)=> inV1[i]+(inV2[i]-inV1[i])*Math.random()), + Box: (inV1, inV2, inCount)=> M.Iterate.Loop(inV1.length, inCount, i=> inV1[i]+(inV2[i]-inV1[i])*Math.random()), Transpose: (inCloud)=> M.Iterate.Loop(inCloud.length, inCloud[0].length, (i, row)=> inCloud[i][row]), Outer: (inV1, inV2)=> M.Iterate.Loop(inV1.length, inV2.length, (i, row)=> inV1[i]*inV2[row]), Clone: (inCloud)=> M.Iterate.Edit(inCloud, i=> i) diff --git a/m.test.js b/m.test.js index 2bbab28..77381c9 100644 --- a/m.test.js +++ b/m.test.js @@ -3,9 +3,9 @@ import { default as M } from "./m.js"; Deno.test("Iterate.Loop", ()=> { - let dimensions = 3; - let count = 4; - let cloud = M.Iterate.Loop(dimensions, count, (i, j)=>i+j); + const dimensions = 3; + const count = 4; + const cloud = M.Iterate.Loop(dimensions, count, (i, j)=>i+j); assertEquals(cloud.length, count, "correct count"); assertEquals(cloud[0].length, dimensions, "correct dimensions"); assertEquals(cloud[0][0], 0); @@ -13,21 +13,20 @@ Deno.test("Iterate.Loop", ()=> }); Deno.test("Iterate.Edit", ()=> { - let c = [[1, 2], [3, 4]] - let t = M.Iterate.Edit(c, (i)=>i); + const c = [[1, 2], [3, 4]] + const t = M.Iterate.Edit(c, (i)=>i); assertEquals(t.length, c.length, "correct count"); assertEquals(t[0][0], c[0][0], "correct dimensions"); assertEquals(t[1][1], c[1][1], "correct placement"); }); - Deno.test("Create.Box", ()=> { - let min = [-1, -2, -3]; - let max = [1, 2, 3]; - let count = 10; + const min = [-1, -2, -3]; + const max = [1, 2, 3]; + const count = 10; - let box = M.Create.Box(min, max, count); + const box = M.Create.Box(min, max, count); assertEquals(box.length, count, "correct count"); for(let i=0; i }); Deno.test("Create.Transpose", ()=> { - let v1 = [1, 2, 3]; - let v2 = [4, 5, 6]; - let tpose = M.Create.Transpose([v1, v2]); + const v1 = [1, 2, 3]; + const v2 = [4, 5, 6]; + const tpose = M.Create.Transpose([v1, v2]); assertEquals(tpose.length, 3, "correct count"); assertEquals(tpose[0].length, 2, "correct dimensions"); assertEquals(tpose[0][0], v1[0]); @@ -51,27 +50,26 @@ Deno.test("Create.Transpose", ()=> }); Deno.test("Create.Outer", ()=> { - let v1 = [1, 2, 3]; - let v2 = [4, 5]; - let outer = M.Create.Outer(v1, v2); + const v1 = [1, 2, 3]; + const v2 = [4, 5]; + const outer = M.Create.Outer(v1, v2); assertEquals(outer.length, v2.length, "correct count"); assertEquals(outer[0].length, v1.length, "correct dimensions"); assertEquals(outer[1][0], v1[0]*v2[1], "correct placement") }); Deno.test("Create.Clone", ()=> { - let v1 = [1, 2, 3]; - let v2 = [4, 5, 6]; - let clone = M.Create.Clone([v1, v2]); + const v1 = [1, 2, 3]; + const v2 = [4, 5, 6]; + const clone = M.Create.Clone([v1, v2]); assertEquals(clone.length, 2, "correct count"); assertEquals(clone[0].length, v1.length, "correct dimensions"); assertEquals(clone[1][0], v2[0], "correct placement"); }); - Deno.test("Mutate.Pad", ()=> { - let matrix = [ + const matrix = [ [1, 2, 3], [4, 5, 6] ]; @@ -80,10 +78,9 @@ Deno.test("Mutate.Pad", ()=> assertEquals(matrix[0].length, 4, "correct dimensions"); assertEquals(matrix[0][3], 1, "correct placement"); }); - Deno.test("Mutate.Unpad", ()=> { - let matrix = [ + const matrix = [ [1, 2, 3, 1], [4, 5, 6, 1] ]; @@ -95,63 +92,54 @@ Deno.test("Mutate.Unpad", ()=> Deno.test("Single.Affine", ()=> { - let v = [1, 2]; - let m = [[0.1, 0.2], [0.3, 0.4]]; - let t = M.Single.Affine(v, m); + const v = [1, 2]; + const m = [[0.1, 0.2], [0.3, 0.4]]; + const t = M.Single.Affine(v, m); assertEquals(t.length, 2, "correct dimensions"); assertEquals(t[0], 0.5) assertEquals(t[1], 1.1, "correct placement"); }); - Deno.test("Single.Subtract", ()=> { - let v1 = [1, 2]; - let v2 = [3, 4]; - let t = M.Single.Subtract(v1, v2); + const v1 = [1, 2]; + const v2 = [3, 4]; + const t = M.Single.Subtract(v1, v2); assertEquals(t.length, 2, "correct dimensions"); assertEquals(t[0], -2) assertEquals(t[1], -2, "correct placement"); }); - Deno.test("Single.Multiply", ()=> { - let v1 = [1, 2]; - let v2 = [3, 4]; - let t = M.Single.Multiply(v1, v2); + const v1 = [1, 2]; + const v2 = [3, 4]; + const t = M.Single.Multiply(v1, v2); assertEquals(t.length, 2, "correct dimensions"); assertEquals(t[0], 3) assertEquals(t[1], 8, "correct placement"); }); - - Deno.test("Batch.Affine", ()=> { - let c = [[1, 2], [3, 4]]; - let m = [[0.1, 0.2], [0.3, 0.4]]; - let t = M.Batch.Affine(c, m); + const c = [[1, 2], [3, 4]]; + const m = [[0.1, 0.2], [0.3, 0.4]]; + const t = M.Batch.Affine(c, m); assertEquals(t.length, 2, "correct count"); assertEquals(t[0].length, 2, "correct dimensions") assertEquals(t[0][1], 1.1, "correct placement"); }); - Deno.test("Batch.Scale", ()=> { - let c = [[1, 2], [3, 4]]; - let s = 0.5; - let t = M.Batch.Scale(c, s); - console.log(t); + const c = [[1, 2], [3, 4]]; + const s = 0.5; + const t = M.Batch.Scale(c, s); assertEquals(t.length, 2, "correct count"); assertEquals(t[0].length, 2, "correct dimensions"); assertEquals(t[1][0], 1.5, "correct placement"); }); - - Deno.test("Batch.Sigmoid", ()=> { - let m = [[-1000, 1000]]; - let t = M.Batch.Sigmoid(m); - console.log(t); + const m = [[-1000, 1000]]; + const t = M.Batch.Sigmoid(m); assertEquals(t.length, 1, "correct count"); assertEquals(t[0].length, 2, "correct dimensions"); assert(t[0][0]>=0 && t[0][0]<0.5); @@ -159,8 +147,8 @@ Deno.test("Batch.Sigmoid", ()=> }); Deno.test("Batch.Derivative", ()=> { - let m = [[-1000, 0, 1000]]; - let t = M.Batch.Derivative(M.Batch.Sigmoid(m)); + const m = [[-1000, 0, 1000]]; + const t = M.Batch.Derivative(M.Batch.Sigmoid(m)); assertEquals(t.length, 1, "correct count"); assertEquals(t[0].length, 3, "correct dimensions"); assert(t[0][0]t[0][2]); From ffb83056175fff39b8458b7b7268f8f31d65685f Mon Sep 17 00:00:00 2001 From: TreetopFlyer Date: Mon, 26 Jul 2021 11:33:07 -0400 Subject: [PATCH 03/13] typescript working --- m.js | 51 --------------------------------------------------- m.test.js | 6 +++--- m.ts | 55 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 58 insertions(+), 54 deletions(-) delete mode 100644 m.js create mode 100644 m.ts diff --git a/m.js b/m.js deleted file mode 100644 index 37a74e5..0000000 --- a/m.js +++ /dev/null @@ -1,51 +0,0 @@ -const M = -{ - Iterate: - { - Loop: (inDimensions, inCount, inFunction)=> - { - let i, j, outputVector; - const outputCloud = []; - for(i=0; row inCloud.map(row=>row.map(inFunction)) - }, - Create: - { - Box: (inV1, inV2, inCount)=> M.Iterate.Loop(inV1.length, inCount, i=> inV1[i]+(inV2[i]-inV1[i])*Math.random()), - Transpose: (inCloud)=> M.Iterate.Loop(inCloud.length, inCloud[0].length, (i, row)=> inCloud[i][row]), - Outer: (inV1, inV2)=> M.Iterate.Loop(inV1.length, inV2.length, (i, row)=> inV1[i]*inV2[row]), - Clone: (inCloud)=> M.Iterate.Edit(inCloud, i=> i) - }, - Mutate: - { - Pad: inCloud=> inCloud.forEach(row=> row.push(1)), - Unpad: inCloud=> inCloud.forEach(row=> row.pop()) - }, - Single: - { - Subtract: (inV1, inV2)=> inV1.map((component, i)=> component-inV2[i]), - Multiply: (inV1, inV2)=> inV1.map((component, i)=> component*inV2[i]), - Affine: (inV, inMatrix)=> inMatrix.map(row=> row.reduce((sum, current, index)=> sum + current*inV[index])) - }, - Batch: - { - Subtract: (inCloud1, inCloud2)=> inCloud1.map((row, rowIndex)=> M.Single.Subtract(row, inCloud2[rowIndex])), - Multiply: (inCloud1, inCloud2)=> inCloud1.map((row, rowIndex)=> M.Single.Multiply(row, inCloud2[rowIndex])), - Affine: (inCloud, inMatrix)=> inCloud.map(row=> M.Single.Affine(row, inMatrix)), - Sigmoid: (inCloud)=> M.Iterate.Edit(inCloud, i=>1/(1+Math.pow(Math.E, -i))), - Derivative: (inCloud)=> M.Iterate.Edit(inCloud, i=>i*(1-i)), - Scale: (inCloud, inScalar)=> M.Iterate.Edit(inCloud, i=>i*inScalar) - } -} - -export default M; \ No newline at end of file diff --git a/m.test.js b/m.test.js index 77381c9..e5e9245 100644 --- a/m.test.js +++ b/m.test.js @@ -1,5 +1,5 @@ import { assert, assertEquals } from "https://deno.land/std@0.102.0/testing/asserts.ts"; -import { default as M } from "./m.js"; +import M from "./m.ts"; Deno.test("Iterate.Loop", ()=> { @@ -33,8 +33,8 @@ Deno.test("Create.Box", ()=> assertEquals(box[i].length, min.length, "correct dimensions"); for(let j=0; j= min[j], true); - assert(box[i][j] <= max[j], true, "correct range"); + assert(box[i][j] >= min[j]); + assert(box[i][j] <= max[j], "correct range"); } } }); diff --git a/m.ts b/m.ts new file mode 100644 index 0000000..6ea74dd --- /dev/null +++ b/m.ts @@ -0,0 +1,55 @@ +export type V = Array; +export type M = Array; +export type HandleLoop = (indexComponent:number, indexRow:number, array:Array) => number; +export type HandleEdit = (component:number, index:number, array:Array) => number; + +const Methods = { + Iterate: + { + Loop: (inDimensions:number, inCount:number, inFunction:HandleLoop):M => + { + let i:number, j:number, outputVector:V; + const outputCloud:M = []; + for(i=0; i inCloud.map((row:V):V=>row.map(inFunction)) + }, + Create: + { + Box: (inV1:V, inV2:V, inCount:number):M=> Methods.Iterate.Loop(inV1.length, inCount, i=> inV1[i]+(inV2[i]-inV1[i])*Math.random()), + Transpose: (inCloud:M):M=> Methods.Iterate.Loop(inCloud.length, inCloud[0].length, (i, row)=> inCloud[i][row]), + Outer: (inV1:V, inV2:V):M=> Methods.Iterate.Loop(inV1.length, inV2.length, (i, row)=> inV1[i]*inV2[row]), + Clone: (inCloud:M):M=> Methods.Iterate.Edit(inCloud, i=> i) + }, + Mutate: + { + Pad: (inCloud:M):void=> inCloud.forEach(row=> row.push(1)), + Unpad: (inCloud:M):void=> inCloud.forEach(row=> row.pop()) + }, + Single: + { + Subtract: (inV1:V, inV2:V):V=> inV1.map((component, i)=> component-inV2[i]), + Multiply: (inV1:V, inV2:V):V=> inV1.map((component, i)=> component*inV2[i]), + Affine: (inV:V, inMatrix:M):V=> inMatrix.map(row=> row.reduce((sum, current, index)=> sum + current*inV[index])) + }, + Batch: + { + Subtract: (inCloud1:M, inCloud2:M):M=> inCloud1.map((row, rowIndex)=> Methods.Single.Subtract(row, inCloud2[rowIndex])), + Multiply: (inCloud1:M, inCloud2:M):M=> inCloud1.map((row, rowIndex)=> Methods.Single.Multiply(row, inCloud2[rowIndex])), + Affine: (inCloud:M, inMatrix:M):M=> inCloud.map(row=> Methods.Single.Affine(row, inMatrix)), + Sigmoid: (inCloud:M):M=> Methods.Iterate.Edit(inCloud, i=>1/(1+Math.pow(Math.E, -i))), + Derivative: (inCloud:M):M=> Methods.Iterate.Edit(inCloud, i=>i*(1-i)), + Scale: (inCloud:M, inScalar:number):M=> Methods.Iterate.Edit(inCloud, i=>i*inScalar) + } +}; + +export default Methods; \ No newline at end of file From 0d06d9e15885c038f2597af8433090d2feaf4d0b Mon Sep 17 00:00:00 2001 From: TreetopFlyer Date: Wed, 28 Jul 2021 12:04:00 -0400 Subject: [PATCH 04/13] Nework added --- m.ts | 51 +++++++++++++++++++++++++--------------------- nn.test.js | 14 +++++++++++++ nn.ts | 36 ++++++++++++++++++++++++++++++++ nn.js => nn_old.js | 0 4 files changed, 78 insertions(+), 23 deletions(-) create mode 100644 nn.test.js create mode 100644 nn.ts rename nn.js => nn_old.js (100%) diff --git a/m.ts b/m.ts index 6ea74dd..f09c07d 100644 --- a/m.ts +++ b/m.ts @@ -1,15 +1,18 @@ -export type V = Array; -export type M = Array; -export type HandleLoop = (indexComponent:number, indexRow:number, array:Array) => number; -export type HandleEdit = (component:number, index:number, array:Array) => number; +export namespace Cloud +{ + export type V = Array + export type M = Array> + export type HandleLoop = (indexComponent:number, indexRow:number, array:Array) => number + export type HandleEdit = (component:number, index:number, array:Array) => number +}; const Methods = { Iterate: { - Loop: (inDimensions:number, inCount:number, inFunction:HandleLoop):M => + Loop: (inDimensions:number, inCount:number, inFunction:Cloud.HandleLoop):Cloud.M => { - let i:number, j:number, outputVector:V; - const outputCloud:M = []; + let i:number, j:number, outputVector:Cloud.V; + const outputCloud:Cloud.M = []; for(i=0; i inCloud.map((row:V):V=>row.map(inFunction)) + Edit: (inCloud:Cloud.M, inFunction:Cloud.HandleEdit):Cloud.M=> inCloud.map((row:Cloud.V):Cloud.V=>row.map(inFunction)) }, Create: { - Box: (inV1:V, inV2:V, inCount:number):M=> Methods.Iterate.Loop(inV1.length, inCount, i=> inV1[i]+(inV2[i]-inV1[i])*Math.random()), - Transpose: (inCloud:M):M=> Methods.Iterate.Loop(inCloud.length, inCloud[0].length, (i, row)=> inCloud[i][row]), - Outer: (inV1:V, inV2:V):M=> Methods.Iterate.Loop(inV1.length, inV2.length, (i, row)=> inV1[i]*inV2[row]), - Clone: (inCloud:M):M=> Methods.Iterate.Edit(inCloud, i=> i) + Box: (inV1:Cloud.V, inV2:Cloud.V, inCount:number):Cloud.M=> Methods.Iterate.Loop(inV1.length, inCount, i=> inV1[i]+(inV2[i]-inV1[i])*Math.random()), + Transpose: (inCloud:Cloud.M):Cloud.M=> Methods.Iterate.Loop(inCloud.length, inCloud[0].length, (i, row)=> inCloud[i][row]), + Outer: (inV1:Cloud.V, inV2:Cloud.V):Cloud.M=> Methods.Iterate.Loop(inV1.length, inV2.length, (i, row)=> inV1[i]*inV2[row]), + Clone: (inCloud:Cloud.M):Cloud.M=> Methods.Iterate.Edit(inCloud, i=> i) }, Mutate: { - Pad: (inCloud:M):void=> inCloud.forEach(row=> row.push(1)), - Unpad: (inCloud:M):void=> inCloud.forEach(row=> row.pop()) + Pad: (inCloud:Cloud.M):Cloud.M=> {inCloud.forEach((row:Cloud.V)=> row.push(1)); return inCloud; }, + Unpad: (inCloud:Cloud.M):Cloud.M=> {inCloud.forEach((row:Cloud.V)=> row.pop()); return inCloud; } }, Single: { - Subtract: (inV1:V, inV2:V):V=> inV1.map((component, i)=> component-inV2[i]), - Multiply: (inV1:V, inV2:V):V=> inV1.map((component, i)=> component*inV2[i]), - Affine: (inV:V, inMatrix:M):V=> inMatrix.map(row=> row.reduce((sum, current, index)=> sum + current*inV[index])) + Subtract: (inV1:Cloud.V, inV2:Cloud.V):Cloud.V=> inV1.map((component, i)=> component-inV2[i]), + Multiply: (inV1:Cloud.V, inV2:Cloud.V):Cloud.V=> inV1.map((component, i)=> component*inV2[i]), + Affine: (inV:Cloud.V, inMatrix:Cloud.M):Cloud.V=> inMatrix.map((row:Cloud.V)=> row.reduce((sum, current, index)=> sum + current*inV[index])) }, Batch: { - Subtract: (inCloud1:M, inCloud2:M):M=> inCloud1.map((row, rowIndex)=> Methods.Single.Subtract(row, inCloud2[rowIndex])), - Multiply: (inCloud1:M, inCloud2:M):M=> inCloud1.map((row, rowIndex)=> Methods.Single.Multiply(row, inCloud2[rowIndex])), - Affine: (inCloud:M, inMatrix:M):M=> inCloud.map(row=> Methods.Single.Affine(row, inMatrix)), - Sigmoid: (inCloud:M):M=> Methods.Iterate.Edit(inCloud, i=>1/(1+Math.pow(Math.E, -i))), - Derivative: (inCloud:M):M=> Methods.Iterate.Edit(inCloud, i=>i*(1-i)), - Scale: (inCloud:M, inScalar:number):M=> Methods.Iterate.Edit(inCloud, i=>i*inScalar) + Subtract: (inCloud1:Cloud.M, inCloud2:Cloud.M):Cloud.M=> inCloud1.map((row:Cloud.V, rowIndex:number)=> Methods.Single.Subtract(row, inCloud2[rowIndex])), + Multiply: (inCloud1:Cloud.M, inCloud2:Cloud.M):Cloud.M=> inCloud1.map((row:Cloud.V, rowIndex:number)=> Methods.Single.Multiply(row, inCloud2[rowIndex])), + Affine: (inCloud1:Cloud.M, inCloud2:Cloud.M):Cloud.M=> inCloud1.map((row:Cloud.V)=> Methods.Single.Affine(row, inCloud2)), + Sigmoid: (inCloud:Cloud.M):Cloud.M=> Methods.Iterate.Edit(inCloud, i=>1/(1+Math.pow(Math.E, -i))), + Derivative: (inCloud:Cloud.M):Cloud.M=> Methods.Iterate.Edit(inCloud, i=>i*(1-i)), + Scale: (inCloud:Cloud.M, inScalar:number):Cloud.M=> Methods.Iterate.Edit(inCloud, i=>i*inScalar) } }; + + export default Methods; \ No newline at end of file diff --git a/nn.test.js b/nn.test.js new file mode 100644 index 0000000..6daee12 --- /dev/null +++ b/nn.test.js @@ -0,0 +1,14 @@ +import { assert, assertEquals } from "https://deno.land/std@0.102.0/testing/asserts.ts"; +import * as NN from "./nn.ts"; + +Deno.test("NN.Observe", ()=> +{ + console.log(NN.Observe([[[1, 2, 3]]], [[[0.4, 0.5, 0.6]]])); +}); +Deno.test("NN.Train", ()=> +{ + const stages = [[[1, 2, 3]]]; + const layers = [[[0.4, 0.5, 0.6]]]; + + console.log(NN.Train(stages, layers, [[0, 0, 1]], 0.1)); +}); diff --git a/nn.ts b/nn.ts new file mode 100644 index 0000000..e02fad0 --- /dev/null +++ b/nn.ts @@ -0,0 +1,36 @@ +import { default as M, Cloud } from "./m.ts"; + +const Observe = (inStages:Array, inLayers:Array):Cloud.M => +{ + let i:number; + let process = (index:number):Cloud.M => M.Batch.Sigmoid(M.Batch.Affine(inStages[index], inLayers[index])); + + for(i=0; i, inLayers:Array, inGoals:Cloud.M, inRate:number):void => +{ + let i:number; + let errorBack:Cloud.M = M.Batch.Subtract(Observe(inStages, inLayers), inGoals); + + for(i=inLayers.length-1; i>=0; i++) + { + let layerMatrix:Cloud.M = inLayers[i]; + let layerInput:Cloud.M = inStages[i]; + let layerOutput:Cloud.M = inStages[i+1]; + let errorScaled:Cloud.M = M.Batch.Multiply(errorBack, M.Batch.Derivative(layerOutput)); + + errorBack = M.Batch.Affine(errorScaled, M.Create.Transpose(layerMatrix)); + errorScaled.forEach((inScaledError:Cloud.V, inIndex:number)=> { + const deltas = M.Batch.Scale(M.Create.Outer(layerInput[inIndex], inScaledError), inRate); + layerMatrix = M.Batch.Subtract(layerMatrix, deltas); + }); + } +}; + +export { Observe, Train }; +export type { Cloud }; \ No newline at end of file diff --git a/nn.js b/nn_old.js similarity index 100% rename from nn.js rename to nn_old.js From 329c5f5f819c5d8a4ebab827b61de42a3aa6bc26 Mon Sep 17 00:00:00 2001 From: TreetopFlyer Date: Wed, 28 Jul 2021 14:08:46 -0400 Subject: [PATCH 05/13] add training set and tests --- nn.test.js | 63 +++++++++++++++++++++++++++++++++++++++++++++++------- nn.ts | 27 ++++++++++++++++++----- 2 files changed, 77 insertions(+), 13 deletions(-) diff --git a/nn.test.js b/nn.test.js index 6daee12..0cda710 100644 --- a/nn.test.js +++ b/nn.test.js @@ -1,14 +1,61 @@ import { assert, assertEquals } from "https://deno.land/std@0.102.0/testing/asserts.ts"; -import * as NN from "./nn.ts"; +import { Label, Forward, Backward } from "./nn.ts"; +import { default as M } from "./m.ts"; -Deno.test("NN.Observe", ()=> +const input = [ + [ 0.1, 0.05], + [ 0.0, -0.06] + [ 0.99, 0.85], + [ 1.2, 1.05] +]; + +const training = []; +const stages = []; +const layers = []; + +Deno.test("NN.Label", ()=> { - console.log(NN.Observe([[[1, 2, 3]]], [[[0.4, 0.5, 0.6]]])); + Label(training, + [ + [ 0.1, 0.05], + [ 0.0, -0.06] + ], + [1]); + Label(training, + [ + [ 0.99, 0.85], + [ 1.2, 1.05] + ], + [0]); + stages.push(training[0]); + console.log(training); + assertEquals(training.length, 2, "input and output sets created"); + assertEquals(training[0].length, training[1].length, "both sets have same length"); + assertEquals(training[0][0].length, 3, "padded input component"); + assertEquals(training[1][0].length, 1, "unchanged label vector"); }); -Deno.test("NN.Train", ()=> + +Deno.test("NN.Backward", ()=> { - const stages = [[[1, 2, 3]]]; - const layers = [[[0.4, 0.5, 0.6]]]; - - console.log(NN.Train(stages, layers, [[0, 0, 1]], 0.1)); + + layers.push(M.Create.Box([0, 0, 0], [1, 1, 1], 4)); + layers.push(M.Create.Box([0, 0, 0, 0, 0], [1, 1, 1, 1, 1], 1)); + + let copy1 = M.Create.Clone(layers[0]); + let copy2 = M.Create.Clone(layers[1]); + + for(let i=0; i<1000; i++) + { + Backward(stages, layers, training[1], 0.1); + } + + assert(layers[0][0][0] != copy1[0][0][0], "first matrix has changed"); + assert(layers[1][0][0] != copy2[0][0][0], "second matrix has changed"); }); + +Deno.test("NN.Forward", ()=> +{ + console.log(Forward(stages, layers)); + console.log(training[1]); +}); + diff --git a/nn.ts b/nn.ts index e02fad0..c08185d 100644 --- a/nn.ts +++ b/nn.ts @@ -1,6 +1,22 @@ import { default as M, Cloud } from "./m.ts"; +export type N = Array>> -const Observe = (inStages:Array, inLayers:Array):Cloud.M => +const Label = (inSet:any, inData:Cloud.M, inLabel:Cloud.V):N => +{ + if(!inSet){inSet = [[], []];} + if(inSet.length == 0){inSet.push([]);} + if(inSet.length == 1){inSet.push([]);} + + inData.forEach((row:Cloud.V) => + { + row.push(1); + inSet[0].push(row); + inSet[1].push(inLabel); + }); + return inSet; +}; + +const Forward = (inStages:N, inLayers:N):Cloud.M => { let i:number; let process = (index:number):Cloud.M => M.Batch.Sigmoid(M.Batch.Affine(inStages[index], inLayers[index])); @@ -12,16 +28,17 @@ const Observe = (inStages:Array, inLayers:Array):Cloud.M => inStages[i+1] = process(i); return inStages[i+1]; }; -const Train = (inStages:Array, inLayers:Array, inGoals:Cloud.M, inRate:number):void => +const Backward = (inStages:N, inLayers:N, inGoals:Cloud.M, inRate:number):void => { let i:number; - let errorBack:Cloud.M = M.Batch.Subtract(Observe(inStages, inLayers), inGoals); + let errorBack:Cloud.M = M.Batch.Subtract(Forward(inStages, inLayers), inGoals); - for(i=inLayers.length-1; i>=0; i++) + for(i=inLayers.length-1; i>=0; i--) { let layerMatrix:Cloud.M = inLayers[i]; let layerInput:Cloud.M = inStages[i]; let layerOutput:Cloud.M = inStages[i+1]; + let errorScaled:Cloud.M = M.Batch.Multiply(errorBack, M.Batch.Derivative(layerOutput)); errorBack = M.Batch.Affine(errorScaled, M.Create.Transpose(layerMatrix)); @@ -32,5 +49,5 @@ const Train = (inStages:Array, inLayers:Array, inGoals:Cloud.M } }; -export { Observe, Train }; +export { Label, Forward, Backward }; export type { Cloud }; \ No newline at end of file From 6c186bc6e47aa1c3da1e563f5b358f3fb1d25527 Mon Sep 17 00:00:00 2001 From: TreetopFlyer Date: Wed, 28 Jul 2021 14:51:49 -0400 Subject: [PATCH 06/13] layers are not being adjusted --- nn.test.js | 33 ++++++++++++++------------------- nn.ts | 1 - 2 files changed, 14 insertions(+), 20 deletions(-) diff --git a/nn.test.js b/nn.test.js index 0cda710..608fd61 100644 --- a/nn.test.js +++ b/nn.test.js @@ -2,16 +2,9 @@ import { assert, assertEquals } from "https://deno.land/std@0.102.0/testing/asse import { Label, Forward, Backward } from "./nn.ts"; import { default as M } from "./m.ts"; -const input = [ - [ 0.1, 0.05], - [ 0.0, -0.06] - [ 0.99, 0.85], - [ 1.2, 1.05] -]; - -const training = []; -const stages = []; -const layers = []; +let training = []; +let stages = []; +let layers = []; Deno.test("NN.Label", ()=> { @@ -38,19 +31,21 @@ Deno.test("NN.Label", ()=> Deno.test("NN.Backward", ()=> { - layers.push(M.Create.Box([0, 0, 0], [1, 1, 1], 4)); - layers.push(M.Create.Box([0, 0, 0, 0, 0], [1, 1, 1, 1, 1], 1)); + let layer1 = M.Create.Box([0, 0, 0], [1, 1, 1], 4); + let layer2 = M.Create.Box([0, 0, 0, 0, 0], [1, 1, 1, 1, 1], 1); + let copy1 = M.Create.Clone(layer1); + let copy2 = M.Create.Clone(layer2); - let copy1 = M.Create.Clone(layers[0]); - let copy2 = M.Create.Clone(layers[1]); + layers.push(layer1); + layers.push(layer2); - for(let i=0; i<1000; i++) + for(let i=0; i<100; i++) { - Backward(stages, layers, training[1], 0.1); + Backward(stages, layers, training[1], 0.01); } - - assert(layers[0][0][0] != copy1[0][0][0], "first matrix has changed"); - assert(layers[1][0][0] != copy2[0][0][0], "second matrix has changed"); + console.log(layer1, copy1); + assert(layer1[0][0] != copy1[0][0], "first matrix has changed"); + assert(layer1[1][0] != copy2[1][0], "second matrix has changed"); }); Deno.test("NN.Forward", ()=> diff --git a/nn.ts b/nn.ts index c08185d..62ec50b 100644 --- a/nn.ts +++ b/nn.ts @@ -38,7 +38,6 @@ const Backward = (inStages:N, inLayers:N, inGoals:Cloud.M, inRate:number):void = let layerMatrix:Cloud.M = inLayers[i]; let layerInput:Cloud.M = inStages[i]; let layerOutput:Cloud.M = inStages[i+1]; - let errorScaled:Cloud.M = M.Batch.Multiply(errorBack, M.Batch.Derivative(layerOutput)); errorBack = M.Batch.Affine(errorScaled, M.Create.Transpose(layerMatrix)); From 574c6704823f31b200e73dc51ac9ec6e18744674 Mon Sep 17 00:00:00 2001 From: TreetopFlyer Date: Wed, 28 Jul 2021 16:40:58 -0400 Subject: [PATCH 07/13] test passing but still bad labels --- m.test.js | 9 +++++++++ nn.test.js | 14 ++++++++------ nn.ts | 10 ++++++---- 3 files changed, 23 insertions(+), 10 deletions(-) diff --git a/m.test.js b/m.test.js index e5e9245..09d9042 100644 --- a/m.test.js +++ b/m.test.js @@ -136,6 +136,15 @@ Deno.test("Batch.Scale", ()=> assertEquals(t[0].length, 2, "correct dimensions"); assertEquals(t[1][0], 1.5, "correct placement"); }); +Deno.test("Batch.Subtract", ()=> +{ + const c = [[1, 2], [3, 4]]; + const s = [[0.5, 0.5], [0.5, 0.5]]; + const t = M.Batch.Subtract(c, s); + assertEquals(t.length, 2, "correct count"); + assertEquals(t[0].length, 2, "correct dimensions"); + assertEquals(t[1][0], 2.5, "correct placement"); +}); Deno.test("Batch.Sigmoid", ()=> { const m = [[-1000, 1000]]; diff --git a/nn.test.js b/nn.test.js index 608fd61..b7e7dff 100644 --- a/nn.test.js +++ b/nn.test.js @@ -31,8 +31,8 @@ Deno.test("NN.Label", ()=> Deno.test("NN.Backward", ()=> { - let layer1 = M.Create.Box([0, 0, 0], [1, 1, 1], 4); - let layer2 = M.Create.Box([0, 0, 0, 0, 0], [1, 1, 1, 1, 1], 1); + let layer1 = M.Create.Box([0, 0, 0], [1, 1, 1], 2); + let layer2 = M.Create.Box([0, 0, 0], [1, 1, 1], 1); let copy1 = M.Create.Clone(layer1); let copy2 = M.Create.Clone(layer2); @@ -41,16 +41,18 @@ Deno.test("NN.Backward", ()=> for(let i=0; i<100; i++) { - Backward(stages, layers, training[1], 0.01); + Backward(stages, layers, training[1], 0.1); } - console.log(layer1, copy1); - assert(layer1[0][0] != copy1[0][0], "first matrix has changed"); - assert(layer1[1][0] != copy2[1][0], "second matrix has changed"); + + assert(layers[0][0][0] != copy1[0][0], "first matrix has changed"); + assert(layers[1][0][0] != copy2[0][0], "second matrix has changed"); }); + Deno.test("NN.Forward", ()=> { console.log(Forward(stages, layers)); console.log(training[1]); }); + diff --git a/nn.ts b/nn.ts index 62ec50b..d902636 100644 --- a/nn.ts +++ b/nn.ts @@ -28,24 +28,26 @@ const Forward = (inStages:N, inLayers:N):Cloud.M => inStages[i+1] = process(i); return inStages[i+1]; }; -const Backward = (inStages:N, inLayers:N, inGoals:Cloud.M, inRate:number):void => +const Backward = (inStages:N, inLayers:N, inGoals:Cloud.M, inRate:number):N => { let i:number; let errorBack:Cloud.M = M.Batch.Subtract(Forward(inStages, inLayers), inGoals); for(i=inLayers.length-1; i>=0; i--) { - let layerMatrix:Cloud.M = inLayers[i]; let layerInput:Cloud.M = inStages[i]; let layerOutput:Cloud.M = inStages[i+1]; let errorScaled:Cloud.M = M.Batch.Multiply(errorBack, M.Batch.Derivative(layerOutput)); - errorBack = M.Batch.Affine(errorScaled, M.Create.Transpose(layerMatrix)); + errorBack = M.Batch.Affine(errorScaled, M.Create.Transpose(inLayers[i])); + errorScaled.forEach((inScaledError:Cloud.V, inIndex:number)=> { const deltas = M.Batch.Scale(M.Create.Outer(layerInput[inIndex], inScaledError), inRate); - layerMatrix = M.Batch.Subtract(layerMatrix, deltas); + inLayers[i] = M.Batch.Subtract(inLayers[i], deltas); }); + } + return inLayers; }; export { Label, Forward, Backward }; From bb9e6e75d05240d6e2c3b8515a552f91f702eb77 Mon Sep 17 00:00:00 2001 From: TreetopFlyer Date: Wed, 28 Jul 2021 20:08:17 -0400 Subject: [PATCH 08/13] debug work --- index.html | 713 +++++++++++++++++++++++++++++++++++++++++++++++++++++ m.test.js | 1 + m.ts | 9 +- nn.test.js | 57 ++++- 4 files changed, 766 insertions(+), 14 deletions(-) create mode 100644 index.html diff --git a/index.html b/index.html new file mode 100644 index 0000000..3a5501d --- /dev/null +++ b/index.html @@ -0,0 +1,713 @@ + + + + + \ No newline at end of file diff --git a/m.test.js b/m.test.js index 09d9042..3183373 100644 --- a/m.test.js +++ b/m.test.js @@ -98,6 +98,7 @@ Deno.test("Single.Affine", ()=> assertEquals(t.length, 2, "correct dimensions"); assertEquals(t[0], 0.5) assertEquals(t[1], 1.1, "correct placement"); + console.log(t); }); Deno.test("Single.Subtract", ()=> { diff --git a/m.ts b/m.ts index f09c07d..a6882d2 100644 --- a/m.ts +++ b/m.ts @@ -38,11 +38,18 @@ const Methods = { Pad: (inCloud:Cloud.M):Cloud.M=> {inCloud.forEach((row:Cloud.V)=> row.push(1)); return inCloud; }, Unpad: (inCloud:Cloud.M):Cloud.M=> {inCloud.forEach((row:Cloud.V)=> row.pop()); return inCloud; } }, + Test: + { + Dot:(v1:Cloud.V, v2:Cloud.V):number=> + { + return v1.reduce((sum, current, index)=> sum + current*v2[index]); + } + }, Single: { Subtract: (inV1:Cloud.V, inV2:Cloud.V):Cloud.V=> inV1.map((component, i)=> component-inV2[i]), Multiply: (inV1:Cloud.V, inV2:Cloud.V):Cloud.V=> inV1.map((component, i)=> component*inV2[i]), - Affine: (inV:Cloud.V, inMatrix:Cloud.M):Cloud.V=> inMatrix.map((row:Cloud.V)=> row.reduce((sum, current, index)=> sum + current*inV[index])) + Affine: (inV:Cloud.V, inMatrix:Cloud.M):Cloud.V=> inMatrix.map((row:Cloud.V)=> row.reduce((sum, current, index)=> sum + current*inV[index], 0)) }, Batch: { diff --git a/nn.test.js b/nn.test.js index b7e7dff..f86af35 100644 --- a/nn.test.js +++ b/nn.test.js @@ -1,25 +1,57 @@ import { assert, assertEquals } from "https://deno.land/std@0.102.0/testing/asserts.ts"; import { Label, Forward, Backward } from "./nn.ts"; import { default as M } from "./m.ts"; +import { default as Methods } from "./m.ts"; let training = []; let stages = []; let layers = []; -Deno.test("NN.Label", ()=> +let typeA = [ + [ 0.1, 0.05], + [ 0.0, -0.06] +]; +let typeB = [ + [ 0.99, 0.85], + [ 1.2, 1.05] +]; + + +Deno.test("check.forward", ()=> { - Label(training, - [ + let training = []; + let stages = []; + let layers = [ + [ + [-0.43662948305036675, -0.368590640707799, -0.23227179558890843], + [-0.004292653969505622, 0.38670055222186317, -0.2478421495365568], + [0.738181366836224, 0.3389203747353555, 0.4920200816404332] + ], + [ + [0.5793881115472015, 0.9732593374796092, 0.15207639877016987, -0.5356575655337803] + ] + ]; + + let typeA = [ [ 0.1, 0.05], [ 0.0, -0.06] - ], - [1]); - Label(training, - [ + ]; + let typeB = [ [ 0.99, 0.85], [ 1.2, 1.05] - ], - [0]); + ]; + + Label(training, typeA, [1]); + stages.push(training[0]); + Forward(stages, layers); + console.log(stages); +}); + +/* +Deno.test("NN.Label", ()=> +{ + Label(training, typeA, [1]); + Label(training, typeB, [0]); stages.push(training[0]); console.log(training); assertEquals(training.length, 2, "input and output sets created"); @@ -30,12 +62,10 @@ Deno.test("NN.Label", ()=> Deno.test("NN.Backward", ()=> { - - let layer1 = M.Create.Box([0, 0, 0], [1, 1, 1], 2); - let layer2 = M.Create.Box([0, 0, 0], [1, 1, 1], 1); + let layer1 = M.Create.Box([-1, -1, -1], [1, 1, 1], 2); + let layer2 = M.Create.Box([-1, -1, -1], [1, 1, 1], 1); let copy1 = M.Create.Clone(layer1); let copy2 = M.Create.Clone(layer2); - layers.push(layer1); layers.push(layer2); @@ -56,3 +86,4 @@ Deno.test("NN.Forward", ()=> }); +*/ \ No newline at end of file From 09053ebf97a86a617282ba2d4e067640c6c46e79 Mon Sep 17 00:00:00 2001 From: TreetopFlyer Date: Thu, 29 Jul 2021 08:42:28 -0400 Subject: [PATCH 09/13] error good --- index.html | 10 ++++++---- nn.test.js | 30 +++++++++++++++--------------- nn.ts | 15 +++++++++------ 3 files changed, 30 insertions(+), 25 deletions(-) diff --git a/index.html b/index.html index 3a5501d..5b7b9a1 100644 --- a/index.html +++ b/index.html @@ -699,15 +699,17 @@ NN.Network.Stochastic = function(inNetwork, inTrainingSet, inIterations) [ 0.99, 0.85], [ 1.2, 1.05] ]; + let goals = [ + [1, 1, 0], + [0, 0, 1] + ]; var layer1 = NN.Layer.Create(1, 1); layer1.Forward.Matrix = matrix1; - var layer2 = NN.Layer.Create(1, 1); - layer2.Forward.Matrix = matrix2; - let stage1 = NN.Layer.Forward(layer1, typeA); + let stage1Error = NN.Layer.Error(layer1, goals); - console.log(stage1); + console.log(stage1Error); \ No newline at end of file diff --git a/nn.test.js b/nn.test.js index f86af35..96de2aa 100644 --- a/nn.test.js +++ b/nn.test.js @@ -19,17 +19,14 @@ let typeB = [ Deno.test("check.forward", ()=> { - let training = []; - let stages = []; - let layers = [ - [ - [-0.43662948305036675, -0.368590640707799, -0.23227179558890843], - [-0.004292653969505622, 0.38670055222186317, -0.2478421495365568], - [0.738181366836224, 0.3389203747353555, 0.4920200816404332] - ], - [ - [0.5793881115472015, 0.9732593374796092, 0.15207639877016987, -0.5356575655337803] - ] + let matrix1 = [ + [-0.43662948305036675, -0.368590640707799, -0.23227179558890843], + [-0.004292653969505622, 0.38670055222186317, -0.2478421495365568], + [0.738181366836224, 0.3389203747353555, 0.4920200816404332] + ]; + + let matrix2 = [ + [0.5793881115472015, 0.9732593374796092, 0.15207639877016987, -0.5356575655337803] ]; let typeA = [ @@ -40,11 +37,14 @@ Deno.test("check.forward", ()=> [ 0.99, 0.85], [ 1.2, 1.05] ]; + let goals = [ + [1, 1, 0], + [0, 0, 1] + ]; - Label(training, typeA, [1]); - stages.push(training[0]); - Forward(stages, layers); - console.log(stages); + let layers = [matrix1]; + let stages = Forward(Methods.Mutate.Pad(typeA), layers); + Backward(stages, layers, goals, 0.1); }); /* diff --git a/nn.ts b/nn.ts index d902636..874b28f 100644 --- a/nn.ts +++ b/nn.ts @@ -16,22 +16,25 @@ const Label = (inSet:any, inData:Cloud.M, inLabel:Cloud.V):N => return inSet; }; -const Forward = (inStages:N, inLayers:N):Cloud.M => +const Forward = (inData:Cloud.M, inLayers:N):N => { let i:number; - let process = (index:number):Cloud.M => M.Batch.Sigmoid(M.Batch.Affine(inStages[index], inLayers[index])); + let stages = [inData]; + let process = (index:number):Cloud.M => M.Batch.Sigmoid(M.Batch.Affine(stages[index], inLayers[index])); for(i=0; i { let i:number; - let errorBack:Cloud.M = M.Batch.Subtract(Forward(inStages, inLayers), inGoals); + let errorBack:Cloud.M = M.Batch.Subtract(inStages[inStages.length-1], inGoals); + + console.log(errorBack); for(i=inLayers.length-1; i>=0; i--) { From 23e71542f2bd78c422b43e9dd3dfe45be0b32f51 Mon Sep 17 00:00:00 2001 From: TreetopFlyer Date: Thu, 29 Jul 2021 09:19:27 -0400 Subject: [PATCH 10/13] adjust ok --- index.html | 5 ++++- nn.ts | 5 +++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/index.html b/index.html index 5b7b9a1..8520882 100644 --- a/index.html +++ b/index.html @@ -709,7 +709,10 @@ NN.Network.Stochastic = function(inNetwork, inTrainingSet, inIterations) let stage1 = NN.Layer.Forward(layer1, typeA); let stage1Error = NN.Layer.Error(layer1, goals); + let stage1Back = NN.Layer.Backward(layer1, stage1Error); - console.log(stage1Error); + console.log("matrix before", layer1.Forward.Matrix); + NN.Layer.Adjust(layer1, 0.1); + console.log("matrix after", layer1.Forward.Matrix); \ No newline at end of file diff --git a/nn.ts b/nn.ts index 874b28f..41789a0 100644 --- a/nn.ts +++ b/nn.ts @@ -34,8 +34,6 @@ const Backward = (inStages:N, inLayers:N, inGoals:Cloud.M, inRate:number):N => let i:number; let errorBack:Cloud.M = M.Batch.Subtract(inStages[inStages.length-1], inGoals); - console.log(errorBack); - for(i=inLayers.length-1; i>=0; i--) { let layerInput:Cloud.M = inStages[i]; @@ -43,12 +41,15 @@ const Backward = (inStages:N, inLayers:N, inGoals:Cloud.M, inRate:number):N => let errorScaled:Cloud.M = M.Batch.Multiply(errorBack, M.Batch.Derivative(layerOutput)); errorBack = M.Batch.Affine(errorScaled, M.Create.Transpose(inLayers[i])); + console.log("matrix before:", inLayers[i]); errorScaled.forEach((inScaledError:Cloud.V, inIndex:number)=> { const deltas = M.Batch.Scale(M.Create.Outer(layerInput[inIndex], inScaledError), inRate); inLayers[i] = M.Batch.Subtract(inLayers[i], deltas); }); + console.log("matrix after:", inLayers[i]); + } return inLayers; }; From 1e644a70dc2614bee8729dcbf4486f9a8c033403 Mon Sep 17 00:00:00 2001 From: TreetopFlyer Date: Thu, 29 Jul 2021 10:13:21 -0400 Subject: [PATCH 11/13] layer batch training ok --- index.html | 42 +++++++++++++++++--------------- nn.test.js | 71 ++++++++++++++++++++++++++++++++---------------------- nn.ts | 3 --- 3 files changed, 64 insertions(+), 52 deletions(-) diff --git a/index.html b/index.html index 8520882..357f297 100644 --- a/index.html +++ b/index.html @@ -614,7 +614,7 @@ NN.Network.Create = function() var i; obj.Layers = []; - obj.LearningRate = 0.8; + obj.LearningRate = 0.1; obj.Error = []; for(i=0; i inNN.Layers.forEach(L=>console.log(L.Forward.Matrix)); - console.log("matrix before", layer1.Forward.Matrix); - NN.Layer.Adjust(layer1, 0.1); - console.log("matrix after", layer1.Forward.Matrix); + logLayers(nn1); + + NN.Network.Batch(nn1, {Input:input, Output:output}, 100); + + logLayers(nn1); \ No newline at end of file diff --git a/nn.test.js b/nn.test.js index 96de2aa..87a2321 100644 --- a/nn.test.js +++ b/nn.test.js @@ -26,48 +26,62 @@ Deno.test("check.forward", ()=> ]; let matrix2 = [ - [0.5793881115472015, 0.9732593374796092, 0.15207639877016987, -0.5356575655337803] + [0.7098703863463034, 0.35485944251238033, 0.7642849892333241, 0.03046174288491077], + [-0.30655426258144347, 0.45509633551425077, -0.5013795222004322, -0.3421292736637427] ]; - let typeA = [ - [ 0.1, 0.05], - [ 0.0, -0.06] + let input = [ + [ 0.1, 0.05], + [ 0.0, -0.06], + [ 0.99, 0.85], + [ 1.2, 1.05] ]; - let typeB = [ - [ 0.99, 0.85], - [ 1.2, 1.05] - ]; - let goals = [ - [1, 1, 0], - [0, 0, 1] + let output = [ + [1, 0], + [1, 0], + [0, 1], + [0, 1] ]; - let layers = [matrix1]; - let stages = Forward(Methods.Mutate.Pad(typeA), layers); - Backward(stages, layers, goals, 0.1); + let layers = [matrix1, matrix2]; + console.log("BEFORE", layers); + for(let i=0; i<100; i++) + { + let stages = Forward(Methods.Mutate.Pad(input), layers); + Backward(stages, layers, output, 0.1); + } + + console.log("AFTER", layers); }); + /* Deno.test("NN.Label", ()=> { - Label(training, typeA, [1]); - Label(training, typeB, [0]); - stages.push(training[0]); - console.log(training); + Label(training, typeA, [1, 0]); + Label(training, typeB, [0, 1]); assertEquals(training.length, 2, "input and output sets created"); assertEquals(training[0].length, training[1].length, "both sets have same length"); assertEquals(training[0][0].length, 3, "padded input component"); - assertEquals(training[1][0].length, 1, "unchanged label vector"); + assertEquals(training[1][0].length, 2, "unchanged label vector"); +}); + +Deno.test("NN.Forward", ()=> +{ + let layer1 = M.Create.Box([-1, -1, -1], [1, 1, 1], 2); + let layer2 = M.Create.Box([-1, -1, -1], [1, 1, 1], 1); + layers.push(layer1); + layers.push(layer2); + + console.log(training[0]); + stages = Forward(training[0], layers); + console.log(stages); }); Deno.test("NN.Backward", ()=> { - let layer1 = M.Create.Box([-1, -1, -1], [1, 1, 1], 2); - let layer2 = M.Create.Box([-1, -1, -1], [1, 1, 1], 1); - let copy1 = M.Create.Clone(layer1); - let copy2 = M.Create.Clone(layer2); - layers.push(layer1); - layers.push(layer2); + let copy1 = M.Create.Clone(layers[0]); + let copy2 = M.Create.Clone(layers[1]); for(let i=0; i<100; i++) { @@ -79,11 +93,10 @@ Deno.test("NN.Backward", ()=> }); -Deno.test("NN.Forward", ()=> +Deno.test("NN.Label", ()=> { - console.log(Forward(stages, layers)); - console.log(training[1]); + let stages = Forward(training[0], layers); + console.log(stages[stages.length-1]); }); - */ \ No newline at end of file diff --git a/nn.ts b/nn.ts index 41789a0..98e666e 100644 --- a/nn.ts +++ b/nn.ts @@ -41,15 +41,12 @@ const Backward = (inStages:N, inLayers:N, inGoals:Cloud.M, inRate:number):N => let errorScaled:Cloud.M = M.Batch.Multiply(errorBack, M.Batch.Derivative(layerOutput)); errorBack = M.Batch.Affine(errorScaled, M.Create.Transpose(inLayers[i])); - console.log("matrix before:", inLayers[i]); errorScaled.forEach((inScaledError:Cloud.V, inIndex:number)=> { const deltas = M.Batch.Scale(M.Create.Outer(layerInput[inIndex], inScaledError), inRate); inLayers[i] = M.Batch.Subtract(inLayers[i], deltas); }); - console.log("matrix after:", inLayers[i]); - } return inLayers; }; From 68aafafc0d8b5141a7825165c5827fe714212188 Mon Sep 17 00:00:00 2001 From: TreetopFlyer Date: Thu, 29 Jul 2021 14:51:00 -0400 Subject: [PATCH 12/13] netowrk functions started --- index.html | 11 +++----- nn.test.js | 23 +++++++++++---- nn.ts | 83 ++++++++++++++++++++++++++++++++++-------------------- 3 files changed, 74 insertions(+), 43 deletions(-) diff --git a/index.html b/index.html index 357f297..9fa0878 100644 --- a/index.html +++ b/index.html @@ -708,13 +708,10 @@ NN.Network.Stochastic = function(inNetwork, inTrainingSet, inIterations) let nn1 = NN.Network.Create(2, 3, 2); nn1.Layers[0].Forward.Matrix = matrix1; nn1.Layers[1].Forward.Matrix = matrix2; + nn1.LearningRate = 0.1; + //let logLayers = inNN => inNN.Layers.forEach(L=>console.log(L.Forward.Matrix)); - let logLayers = inNN => inNN.Layers.forEach(L=>console.log(L.Forward.Matrix)); - - logLayers(nn1); - - NN.Network.Batch(nn1, {Input:input, Output:output}, 100); - - logLayers(nn1); + NN.Network.Batch(nn1, {Input:input, Output:output}, 1000); + console.log(NN.Network.Observe(nn1, input)); \ No newline at end of file diff --git a/nn.test.js b/nn.test.js index 87a2321..c5d7e7c 100644 --- a/nn.test.js +++ b/nn.test.js @@ -1,5 +1,5 @@ import { assert, assertEquals } from "https://deno.land/std@0.102.0/testing/asserts.ts"; -import { Label, Forward, Backward } from "./nn.ts"; +import { Split, Forward, Backward } from "./nn.ts"; import { default as M } from "./m.ts"; import { default as Methods } from "./m.ts"; @@ -17,7 +17,7 @@ let typeB = [ ]; -Deno.test("check.forward", ()=> +Deno.test("check forward/backward", ()=> { let matrix1 = [ [-0.43662948305036675, -0.368590640707799, -0.23227179558890843], @@ -44,14 +44,25 @@ Deno.test("check.forward", ()=> ]; let layers = [matrix1, matrix2]; - console.log("BEFORE", layers); - for(let i=0; i<100; i++) + let stages = []; + for(let i=0; i<1000; i++) { - let stages = Forward(Methods.Mutate.Pad(input), layers); + stages = Forward(Methods.Mutate.Pad(input), layers); Backward(stages, layers, output, 0.1); } - console.log("AFTER", layers); + stages = Forward(input, layers); + console.log(stages[stages.length-1]); +}); + +Deno.test("NN.Split", ()=> +{ + let data = [ + [3, 2, 1, 0, 1], + [6, 5, 4, 1, 0] + ] + let split = Split(data, [3, 4]); + console.log(split); }); diff --git a/nn.ts b/nn.ts index 98e666e..c7922a7 100644 --- a/nn.ts +++ b/nn.ts @@ -1,31 +1,13 @@ import { default as M, Cloud } from "./m.ts"; export type N = Array>> -const Label = (inSet:any, inData:Cloud.M, inLabel:Cloud.V):N => -{ - if(!inSet){inSet = [[], []];} - if(inSet.length == 0){inSet.push([]);} - if(inSet.length == 1){inSet.push([]);} - - inData.forEach((row:Cloud.V) => - { - row.push(1); - inSet[0].push(row); - inSet[1].push(inLabel); - }); - return inSet; -}; - const Forward = (inData:Cloud.M, inLayers:N):N => { let i:number; - let stages = [inData]; + let stages:N = [inData]; let process = (index:number):Cloud.M => M.Batch.Sigmoid(M.Batch.Affine(stages[index], inLayers[index])); - for(i=0; i for(i=inLayers.length-1; i>=0; i--) { - let layerInput:Cloud.M = inStages[i]; - let layerOutput:Cloud.M = inStages[i+1]; - let errorScaled:Cloud.M = M.Batch.Multiply(errorBack, M.Batch.Derivative(layerOutput)); - + let errorScaled:Cloud.M = M.Batch.Multiply(errorBack, M.Batch.Derivative(inStages[i+1])); errorBack = M.Batch.Affine(errorScaled, M.Create.Transpose(inLayers[i])); - - errorScaled.forEach((inScaledError:Cloud.V, inIndex:number)=> { - const deltas = M.Batch.Scale(M.Create.Outer(layerInput[inIndex], inScaledError), inRate); - inLayers[i] = M.Batch.Subtract(inLayers[i], deltas); + errorScaled.forEach((inScaledError:Cloud.V, inIndex:number)=> + { + inLayers[i] = M.Batch.Subtract( + inLayers[i], + M.Batch.Scale(M.Create.Outer(inStages[i][inIndex], inScaledError), inRate) + ); }); - } return inLayers; }; +const Split = (inTrainingSet:Cloud.M, inHeaderLabel:Cloud.V, inHeaderKeep:Cloud.V = []):N => +{ + let data:Cloud.M = []; + let label:Cloud.M = []; + if(!inHeaderKeep.length) + { + inTrainingSet[0].forEach( (item:number, index:number)=> inHeaderLabel.includes(index) ? false : inHeaderKeep.push(index) ); + } + inTrainingSet.forEach((row:Cloud.V) => + { + data.push( [...inHeaderKeep.map((i:number)=>row[i]), 1] ); + label.push( inHeaderLabel.map((i:number)=>row[i]) ); + }); + return [ data, label ]; +}; +const Build = (...inLayers:Array):N => +{ + let i:number; + let output:N = []; + let rand = (inDimensions:number, inCount:number):Cloud.M => M.Create.Box( new Array(inDimensions).fill(-1), new Array(inDimensions).fill(1), inCount); + for(i=0; i +{ + let stages:N = Forward(inData, inLayers); + return stages[stages.length-1]; +}; +const Learn = (inData:Cloud.M, inLayers:N, inLabels:Cloud.M, inIterations:number, inRate:number):Cloud.M => +{ + let stages:N = []; + for(let i=0; i Date: Thu, 29 Jul 2021 16:15:05 -0400 Subject: [PATCH 13/13] network tests passing --- nn.test.js | 135 +++++++++++++++-------------------------------------- nn.ts | 14 +++--- 2 files changed, 44 insertions(+), 105 deletions(-) diff --git a/nn.test.js b/nn.test.js index c5d7e7c..6519caf 100644 --- a/nn.test.js +++ b/nn.test.js @@ -1,113 +1,52 @@ import { assert, assertEquals } from "https://deno.land/std@0.102.0/testing/asserts.ts"; -import { Split, Forward, Backward } from "./nn.ts"; -import { default as M } from "./m.ts"; -import { default as Methods } from "./m.ts"; +import { Split, Build, Label, Learn, Check } from "./nn.ts"; -let training = []; -let stages = []; +let data = [ + [ 0.10, 0.05, 0, 1], + [ 0.00, -0.06, 0, 1], + [ 0.99, 0.85, 1, 0], + [ 1.20, 1.05, 1, 0] +]; +let columns = [2, 3]; +let input, output; let layers = []; -let typeA = [ - [ 0.1, 0.05], - [ 0.0, -0.06] -]; -let typeB = [ - [ 0.99, 0.85], - [ 1.2, 1.05] -]; - - -Deno.test("check forward/backward", ()=> -{ - let matrix1 = [ - [-0.43662948305036675, -0.368590640707799, -0.23227179558890843], - [-0.004292653969505622, 0.38670055222186317, -0.2478421495365568], - [0.738181366836224, 0.3389203747353555, 0.4920200816404332] - ]; - - let matrix2 = [ - [0.7098703863463034, 0.35485944251238033, 0.7642849892333241, 0.03046174288491077], - [-0.30655426258144347, 0.45509633551425077, -0.5013795222004322, -0.3421292736637427] - ]; - - let input = [ - [ 0.1, 0.05], - [ 0.0, -0.06], - [ 0.99, 0.85], - [ 1.2, 1.05] - ]; - let output = [ - [1, 0], - [1, 0], - [0, 1], - [0, 1] - ]; - - let layers = [matrix1, matrix2]; - let stages = []; - for(let i=0; i<1000; i++) - { - stages = Forward(Methods.Mutate.Pad(input), layers); - Backward(stages, layers, output, 0.1); - } - - stages = Forward(input, layers); - console.log(stages[stages.length-1]); -}); - Deno.test("NN.Split", ()=> { - let data = [ - [3, 2, 1, 0, 1], - [6, 5, 4, 1, 0] - ] - let split = Split(data, [3, 4]); - console.log(split); + [input, output] = Split(data, columns); + assert(input); + assert(output); + assertEquals(input.length, output.length, "data split into equal input and output"); + + assertEquals(input[0].length, 3, "padded input"); + assertEquals(output[0].length, 2, "unpadded output"); }); - -/* -Deno.test("NN.Label", ()=> +Deno.test("NN.Build", ()=> { - Label(training, typeA, [1, 0]); - Label(training, typeB, [0, 1]); - assertEquals(training.length, 2, "input and output sets created"); - assertEquals(training[0].length, training[1].length, "both sets have same length"); - assertEquals(training[0][0].length, 3, "padded input component"); - assertEquals(training[1][0].length, 2, "unchanged label vector"); + layers = Build(2, 5, 2); + + assertEquals(layers.length, 2, "correct number of matrices"); + assertEquals(layers[0][0].length, input[0].length, "input: padded input"); + assertEquals(layers[0].length, 5, "input: unpadded output"); + + assertEquals(layers[1][0].length, 6, "hidden: padded input"); + assertEquals(layers[1].length, output[0].length, "hidden: unpadded output"); }); -Deno.test("NN.Forward", ()=> -{ - let layer1 = M.Create.Box([-1, -1, -1], [1, 1, 1], 2); - let layer2 = M.Create.Box([-1, -1, -1], [1, 1, 1], 1); - layers.push(layer1); - layers.push(layer2); - - console.log(training[0]); - stages = Forward(training[0], layers); - console.log(stages); -}); - -Deno.test("NN.Backward", ()=> -{ - let copy1 = M.Create.Clone(layers[0]); - let copy2 = M.Create.Clone(layers[1]); - - for(let i=0; i<100; i++) - { - Backward(stages, layers, training[1], 0.1); - } - - assert(layers[0][0][0] != copy1[0][0], "first matrix has changed"); - assert(layers[1][0][0] != copy2[0][0], "second matrix has changed"); -}); - - Deno.test("NN.Label", ()=> { - let stages = Forward(training[0], layers); - console.log(stages[stages.length-1]); + let labels = Label(input, layers); + assertEquals(labels.length, output.length); + assertEquals(labels[0].length, output[0].length); }); -*/ \ No newline at end of file +Deno.test("NN.Learn", ()=> +{ + let error = Learn(input, layers, output, 1000, 0.1); + assertEquals(error.length, output.length); + let total = 0; + let count = error.length*error[0].length; + error.forEach(row=> row.forEach(component=> total+=Math.abs(component))); + assert(total/count < 0.3); +}); \ No newline at end of file diff --git a/nn.ts b/nn.ts index c7922a7..a09e8a1 100644 --- a/nn.ts +++ b/nn.ts @@ -38,10 +38,12 @@ const Split = (inTrainingSet:Cloud.M, inHeaderLabel:Cloud.V, inHeaderKeep:Cloud. { inTrainingSet[0].forEach( (item:number, index:number)=> inHeaderLabel.includes(index) ? false : inHeaderKeep.push(index) ); } - inTrainingSet.forEach((row:Cloud.V) => + inTrainingSet.forEach((row:Cloud.V):void => { - data.push( [...inHeaderKeep.map((i:number)=>row[i]), 1] ); - label.push( inHeaderLabel.map((i:number)=>row[i]) ); + let vectorData = [ ...inHeaderKeep.map((i:number)=>row[i]), 1]; + let vectorLabel = inHeaderLabel.map((i:number)=>row[i]) + data.push( vectorData ); + label.push( vectorLabel ); }); return [ data, label ]; }; @@ -54,7 +56,6 @@ const Build = (...inLayers:Array):N => { output.push(rand( inLayers[i]+1, inLayers[i+1])); } - output.push( rand( inLayers[i-1], inLayers[i]) ); return output; }; const Label = (inData:Cloud.M, inLayers:N):Cloud.M => @@ -72,7 +73,6 @@ const Learn = (inData:Cloud.M, inLayers:N, inLabels:Cloud.M, inIterations:number } return M.Batch.Subtract(stages[stages.length-1], inLabels); }; -const Error = M.Batch.Subtract; +const Check = (inData:Cloud.M, inLayers:N, inLabels:Cloud.M):Cloud.M => Learn(inData, inLayers, inLabels, 1, 0); -export { Split, Build, Label, Learn, Error, Forward, Backward }; -export type { Cloud }; \ No newline at end of file +export { Split, Build, Label, Learn, Check, Forward, Backward }; \ No newline at end of file