Long Short-Term Memory (LSTM) is a module of Recurrent Neural Networks (RNN) that is used for processing sequences of data. It is commonly used for time series prediction. LSTM units are composed of a cell, input gate, output gate, and forget gate, which regulate the flow of information into and out of the cell. LSTM units can remember values over arbitrary time intervals. Gated Recurrent Units (GRU) are a variant of LSTM that have a simpler architecture and are believed to be more time-efficient. GRU units are composed of a reset gate, update gate, and new gate. The time series prediction model in PAL (the context in which this content is presented) consists of a multi-layer LSTM/GRU and a fully connected layer.
------

SET SCHEMA DM_PAL;

DROP TABLE PAL_PARAMETER_TBL;

CREATE COLUMN TABLE PAL_PARAMETER_TBL (PARAM_NAME VARCHAR(256), INT_VALUE INTEGER, DOUBLE_VALUE DOUBLE, STRING_VALUE VARCHAR(1000));

INSERT INTO PAL_PARAMETER_TBL VALUES ('GRU',               0,    Null,   Null);
INSERT INTO PAL_PARAMETER_TBL VALUES ('BIDIRECTIONAL',     0,    Null,   Null);
INSERT INTO PAL_PARAMETER_TBL VALUES ('TIME_DIM',          6,    Null,   Null);
INSERT INTO PAL_PARAMETER_TBL VALUES ('MAX_ITER',       1000,    Null,   Null);
INSERT INTO PAL_PARAMETER_TBL VALUES ('LEARNING_RATE',  Null,    0.01,   Null);
INSERT INTO PAL_PARAMETER_TBL VALUES ('BATCH_SIZE',        8,    Null,   Null);
INSERT INTO PAL_PARAMETER_TBL VALUES ('HIDDEN_DIM',      128,    Null,   Null);
INSERT INTO PAL_PARAMETER_TBL VALUES ('NUM_LAYERS',        1,    Null,   Null);
INSERT INTO PAL_PARAMETER_TBL VALUES ('INTERVAL',         50,    Null,   Null);
INSERT INTO PAL_PARAMETER_TBL VALUES ('STATEFUL',          0,    Null,   Null);
INSERT INTO PAL_PARAMETER_TBL VALUES ('OPTIMIZER_TYPE',    2,    Null,   Null);

DROP TABLE PAL_LSTM_DATA_TBL;
CREATE COLUMN TABLE PAL_LSTM_DATA_TBL (
	TIME_STAMP INTEGER,
	TARGET DOUBLE
);

INSERT INTO PAL_LSTM_DATA_TBL VALUES (1, 129.0);
INSERT INTO PAL_LSTM_DATA_TBL VALUES (2, 148.0);
INSERT INTO PAL_LSTM_DATA_TBL VALUES (3, 159.0);
INSERT INTO PAL_LSTM_DATA_TBL VALUES (4, 181.0);
INSERT INTO PAL_LSTM_DATA_TBL VALUES (5, 138.0);
INSERT INTO PAL_LSTM_DATA_TBL VALUES (6, 109.0);
INSERT INTO PAL_LSTM_DATA_TBL VALUES (7, 105.0);
INSERT INTO PAL_LSTM_DATA_TBL VALUES (8, 124.0);
INSERT INTO PAL_LSTM_DATA_TBL VALUES (9, 120.0);
INSERT INTO PAL_LSTM_DATA_TBL VALUES (10, 132.0);
INSERT INTO PAL_LSTM_DATA_TBL VALUES (11, 140.0);
INSERT INTO PAL_LSTM_DATA_TBL VALUES (12, 152.0);
INSERT INTO PAL_LSTM_DATA_TBL VALUES (13, 148.0);
INSERT INTO PAL_LSTM_DATA_TBL VALUES (14, 164.0);
INSERT INTO PAL_LSTM_DATA_TBL VALUES (15, 158.0);
INSERT INTO PAL_LSTM_DATA_TBL VALUES (16, 154.0);
INSERT INTO PAL_LSTM_DATA_TBL VALUES (17, 159.0);
INSERT INTO PAL_LSTM_DATA_TBL VALUES (18, 164.0);
INSERT INTO PAL_LSTM_DATA_TBL VALUES (19, 170.0);
INSERT INTO PAL_LSTM_DATA_TBL VALUES (20, 149.0);
INSERT INTO PAL_LSTM_DATA_TBL VALUES (21, 154.0);
INSERT INTO PAL_LSTM_DATA_TBL VALUES (22, 164.0);
INSERT INTO PAL_LSTM_DATA_TBL VALUES (23, 156.0);
INSERT INTO PAL_LSTM_DATA_TBL VALUES (24, 126.0);
INSERT INTO PAL_LSTM_DATA_TBL VALUES (25, 90.0);
INSERT INTO PAL_LSTM_DATA_TBL VALUES (26, 63.0);
INSERT INTO PAL_LSTM_DATA_TBL VALUES (27, 65.0);
INSERT INTO PAL_LSTM_DATA_TBL VALUES (28, 55.0);
INSERT INTO PAL_LSTM_DATA_TBL VALUES (29, 65.0);
INSERT INTO PAL_LSTM_DATA_TBL VALUES (30, 83.0);
INSERT INTO PAL_LSTM_DATA_TBL VALUES (31, 91.0);
INSERT INTO PAL_LSTM_DATA_TBL VALUES (32, 86.0);
INSERT INTO PAL_LSTM_DATA_TBL VALUES (33, 82.0);
INSERT INTO PAL_LSTM_DATA_TBL VALUES (34, 86.0);
INSERT INTO PAL_LSTM_DATA_TBL VALUES (35, 78.0);
INSERT INTO PAL_LSTM_DATA_TBL VALUES (36, 98.0);
INSERT INTO PAL_LSTM_DATA_TBL VALUES (37, 107.0);
INSERT INTO PAL_LSTM_DATA_TBL VALUES (38, 90.0);
INSERT INTO PAL_LSTM_DATA_TBL VALUES (39, 96.0);
INSERT INTO PAL_LSTM_DATA_TBL VALUES (40, 95.0);
INSERT INTO PAL_LSTM_DATA_TBL VALUES (41, 86.0);
INSERT INTO PAL_LSTM_DATA_TBL VALUES (42, 70.0);
INSERT INTO PAL_LSTM_DATA_TBL VALUES (43, 61.0);
INSERT INTO PAL_LSTM_DATA_TBL VALUES (44, 53.0);
INSERT INTO PAL_LSTM_DATA_TBL VALUES (45, 71.0);
INSERT INTO PAL_LSTM_DATA_TBL VALUES (46, 72.0);
INSERT INTO PAL_LSTM_DATA_TBL VALUES (47, 76.0);
INSERT INTO PAL_LSTM_DATA_TBL VALUES (48, 73.0);
INSERT INTO PAL_LSTM_DATA_TBL VALUES (49, 79.0);
INSERT INTO PAL_LSTM_DATA_TBL VALUES (50, 58.0);
INSERT INTO PAL_LSTM_DATA_TBL VALUES (51, 25.0);
INSERT INTO PAL_LSTM_DATA_TBL VALUES (52, 26.0);
INSERT INTO PAL_LSTM_DATA_TBL VALUES (53, 28.0);
INSERT INTO PAL_LSTM_DATA_TBL VALUES (54, 26.0);
INSERT INTO PAL_LSTM_DATA_TBL VALUES (55, 20.0);
INSERT INTO PAL_LSTM_DATA_TBL VALUES (56, 29.0);
INSERT INTO PAL_LSTM_DATA_TBL VALUES (57, 26.0);
INSERT INTO PAL_LSTM_DATA_TBL VALUES (58, 27.0);
INSERT INTO PAL_LSTM_DATA_TBL VALUES (59, 27.0);
INSERT INTO PAL_LSTM_DATA_TBL VALUES (60, 25.0);
INSERT INTO PAL_LSTM_DATA_TBL VALUES (61, 29.0);
INSERT INTO PAL_LSTM_DATA_TBL VALUES (62, 32.0);
INSERT INTO PAL_LSTM_DATA_TBL VALUES (63, 28.0);
INSERT INTO PAL_LSTM_DATA_TBL VALUES (64, 29.0);
INSERT INTO PAL_LSTM_DATA_TBL VALUES (65, 30.0);
INSERT INTO PAL_LSTM_DATA_TBL VALUES (66, 30.0);
INSERT INTO PAL_LSTM_DATA_TBL VALUES (67, 28.0);
INSERT INTO PAL_LSTM_DATA_TBL VALUES (68, 26.0);
INSERT INTO PAL_LSTM_DATA_TBL VALUES (69, 31.0);
INSERT INTO PAL_LSTM_DATA_TBL VALUES (70, 33.0);
INSERT INTO PAL_LSTM_DATA_TBL VALUES (71, 29.0);
INSERT INTO PAL_LSTM_DATA_TBL VALUES (72, 31.0);
INSERT INTO PAL_LSTM_DATA_TBL VALUES (73, 30.0);
INSERT INTO PAL_LSTM_DATA_TBL VALUES (74, 34.0);
INSERT INTO PAL_LSTM_DATA_TBL VALUES (75, 27.0);
INSERT INTO PAL_LSTM_DATA_TBL VALUES (76, 25.0);
INSERT INTO PAL_LSTM_DATA_TBL VALUES (77, 28.0);
INSERT INTO PAL_LSTM_DATA_TBL VALUES (78, 28.0);
INSERT INTO PAL_LSTM_DATA_TBL VALUES (79, 27.0);
INSERT INTO PAL_LSTM_DATA_TBL VALUES (80, 27.0);
INSERT INTO PAL_LSTM_DATA_TBL VALUES (81, 27.0);
INSERT INTO PAL_LSTM_DATA_TBL VALUES (82, 29.0);
INSERT INTO PAL_LSTM_DATA_TBL VALUES (83, 36.0);
INSERT INTO PAL_LSTM_DATA_TBL VALUES (84, 30.0);
INSERT INTO PAL_LSTM_DATA_TBL VALUES (85, 27.0);
INSERT INTO PAL_LSTM_DATA_TBL VALUES (86, 39.0);
INSERT INTO PAL_LSTM_DATA_TBL VALUES (87, 41.0);
INSERT INTO PAL_LSTM_DATA_TBL VALUES (88, 33.0);
INSERT INTO PAL_LSTM_DATA_TBL VALUES (89, 50.0);
INSERT INTO PAL_LSTM_DATA_TBL VALUES (90, 56.0);
INSERT INTO PAL_LSTM_DATA_TBL VALUES (91, 59.0);
INSERT INTO PAL_LSTM_DATA_TBL VALUES (92, 60.0);
INSERT INTO PAL_LSTM_DATA_TBL VALUES (93, 84.0);
INSERT INTO PAL_LSTM_DATA_TBL VALUES (94, 106.0);
INSERT INTO PAL_LSTM_DATA_TBL VALUES (95, 66.0);
INSERT INTO PAL_LSTM_DATA_TBL VALUES (96, 50.0);
INSERT INTO PAL_LSTM_DATA_TBL VALUES (97, 56.0);
INSERT INTO PAL_LSTM_DATA_TBL VALUES (98, 77.0);
INSERT INTO PAL_LSTM_DATA_TBL VALUES (99, 50.0);
INSERT INTO PAL_LSTM_DATA_TBL VALUES (100, 44.0);

DROP TABLE PAL_LSTM_MODEL_TBL; -- for predict followed
CREATE COLUMN TABLE PAL_LSTM_MODEL_TBL ( 
	ROW_INDEX INTEGER,
	MODEL_CONTENT NVARCHAR(5000)
);

DROP TABLE PAL_LSTM_LOSS_TBL; 
CREATE COLUMN TABLE PAL_LSTM_LOSS_TBL ( 
	ITER INTEGER,
	LOSS DOUBLE
);


DO BEGIN
	lt_data = SELECT * FROM PAL_LSTM_DATA_TBL;
	lt_param = SELECT * FROM PAL_PARAMETER_TBL;
	CALL _SYS_AFL.PAL_LSTM_TRAIN (:lt_data, :lt_param, lt_loss, lt_model);
	INSERT INTO PAL_LSTM_LOSS_TBL SELECT * FROM :lt_loss;
	INSERT INTO PAL_LSTM_MODEL_TBL SELECT * FROM :lt_model;
END;

select * from PAL_LSTM_LOSS_TBL;
select * from PAL_LSTM_MODEL_TBL;
