Initial commit.

.gitignore

@@ -0,0 +1,7 @@
+*.zip
+__pycache__/
+*.pth
+*.log
+*.aux
+*.synctex.gz
+*.synctex.gz(buzy)

hw1/.vscode/settings.json

@@ -0,0 +1,4 @@
+{
+    "python.analysis.typeCheckingMode": "basic",
+    "python.analysis.autoImportCompletions": true
+}

hw1/HW1-Report/codes/1.1.in.txt

@@ -0,0 +1 @@
+python recognition.py --mode train --model_path ./saved_models/default.pth

hw1/HW1-Report/codes/1.1.out.txt

@@ -0,0 +1,56 @@
+Epoch 01: loss = inf
+Epoch 02: loss = inf
+Epoch 03: loss = 6.678
+Epoch 04: loss = 4.361
+Epoch 05: loss = 3.110
+Epoch 06: loss = 2.099
+Epoch 07: loss = 1.698
+Epoch 08: loss = 1.320
+Epoch 09: loss = 0.970
+Epoch 10: loss = 0.891
+Epoch 10: validation accuracy = 66.0%
+Epoch 11: loss = 0.817
+Epoch 12: loss = 0.723
+Epoch 13: loss = 0.512
+Epoch 14: loss = 0.353
+Epoch 15: loss = 0.202
+Epoch 16: loss = 0.182
+Epoch 17: loss = 0.184
+Epoch 18: loss = 0.191
+Epoch 19: loss = 0.175
+Epoch 20: loss = 0.166
+Epoch 20: validation accuracy = 68.0%
+Epoch 21: loss = 0.146
+Epoch 22: loss = 0.105
+Epoch 23: loss = 0.109
+Epoch 24: loss = 0.074
+Epoch 25: loss = 0.097
+Epoch 26: loss = 0.047
+Epoch 27: loss = 0.038
+Epoch 28: loss = 0.037
+Epoch 29: loss = 0.024
+Epoch 30: loss = 0.021
+Epoch 30: validation accuracy = 68.8%
+Epoch 31: loss = 0.019
+Epoch 32: loss = 0.024
+Epoch 33: loss = 0.023
+Epoch 34: loss = 0.014
+Epoch 35: loss = 0.013
+Epoch 36: loss = 0.012
+Epoch 37: loss = 0.011
+Epoch 38: loss = 0.013
+Epoch 39: loss = 0.013
+Epoch 40: loss = 0.016
+Epoch 40: validation accuracy = 70.5%
+Epoch 41: loss = 0.015
+Epoch 42: loss = 0.009
+Epoch 43: loss = 0.011
+Epoch 44: loss = 0.008
+Epoch 45: loss = 0.008
+Epoch 46: loss = 0.010
+Epoch 47: loss = 0.009
+Epoch 48: loss = 0.007
+Epoch 49: loss = 0.007
+Epoch 50: loss = 0.010
+Epoch 50: validation accuracy = 70.5%
+Model saved in ./saved_models/default.pth

hw1/HW1-Report/codes/1.2.in.txt

@@ -0,0 +1 @@
+python recognition.py --mode test --model_path .\saved_models\default.pth

hw1/HW1-Report/codes/1.2.out.txt

@@ -0,0 +1,2 @@
+[Info] Load model from .\saved_models\default.pth
+[Info] Test accuracy = 72.0%

hw1/HW1-Report/codes/2.1.in.txt

@@ -0,0 +1 @@
+python recognition.py --mode train --hsize 64 --lr 2e-3 --optim_type adam --momentum 0 --weight_decay 0.1 --model_path .\saved_models\adam_optim.pth

hw1/HW1-Report/codes/2.1.out.txt

@@ -0,0 +1,56 @@
+Epoch 01: loss = inf
+Epoch 02: loss = inf
+Epoch 03: loss = inf
+Epoch 04: loss = inf
+Epoch 05: loss = inf
+Epoch 06: loss = inf
+Epoch 07: loss = inf
+Epoch 08: loss = inf
+Epoch 09: loss = 3.286
+Epoch 10: loss = 2.528
+Epoch 10: validation accuracy = 57.5%
+Epoch 11: loss = 1.864
+Epoch 12: loss = 1.593
+Epoch 13: loss = 1.416
+Epoch 14: loss = 0.983
+Epoch 15: loss = 0.798
+Epoch 16: loss = 0.986
+Epoch 17: loss = 0.870
+Epoch 18: loss = 0.589
+Epoch 19: loss = 0.821
+Epoch 20: loss = 0.957
+Epoch 20: validation accuracy = 66.2%
+Epoch 21: loss = 0.557
+Epoch 22: loss = 0.829
+Epoch 23: loss = 0.699
+Epoch 24: loss = 0.647
+Epoch 25: loss = 0.507
+Epoch 26: loss = 0.693
+Epoch 27: loss = 0.628
+Epoch 28: loss = 0.414
+Epoch 29: loss = 0.840
+Epoch 30: loss = 0.543
+Epoch 30: validation accuracy = 66.8%
+Epoch 31: loss = 0.816
+Epoch 32: loss = 0.483
+Epoch 33: loss = 0.476
+Epoch 34: loss = 0.523
+Epoch 35: loss = 0.578
+Epoch 36: loss = 0.583
+Epoch 37: loss = 0.375
+Epoch 38: loss = 0.428
+Epoch 39: loss = 0.678
+Epoch 40: loss = 0.499
+Epoch 40: validation accuracy = 72.2%
+Epoch 41: loss = 0.436
+Epoch 42: loss = 0.488
+Epoch 43: loss = 0.650
+Epoch 44: loss = 0.412
+Epoch 45: loss = 0.249
+Epoch 46: loss = 0.404
+Epoch 47: loss = 0.405
+Epoch 48: loss = 0.482
+Epoch 49: loss = 0.381
+Epoch 50: loss = 0.358
+Epoch 50: validation accuracy = 79.5%
+Model saved in .\saved_models\adam_optim.pth

hw1/HW1-Report/codes/2.2.in.txt

@@ -0,0 +1 @@
+python recognition.py --mode test --hsize 64 --model_path .\saved_models\adam_optim.pth

hw1/HW1-Report/codes/2.2.out.txt

@@ -0,0 +1,2 @@
+[Info] Load model from .\saved_models\adam_optim.pth
+[Info] Test accuracy = 85.0%

hw1/HW1-Report/codes/ada_optim_cuda.in.txt

@@ -0,0 +1 @@
+python recognition.py --mode train --hsize 64 --lr 2e-3 --optim_type adam --momentum 0 --weight_decay 0.1 --model_path .\saved_models\adam_optim_cuda.pth --device cuda

hw1/HW1-Report/codes/adam_optim_cuda.out.txt

@@ -0,0 +1,56 @@
+Epoch 01: loss = inf
+Epoch 02: loss = inf
+Epoch 03: loss = inf
+Epoch 04: loss = inf
+Epoch 05: loss = inf
+Epoch 06: loss = inf
+Epoch 07: loss = inf
+Epoch 08: loss = inf
+Epoch 09: loss = 3.250
+Epoch 10: loss = 2.567
+Epoch 10: validation accuracy = 59.0%
+Epoch 11: loss = 1.963
+Epoch 12: loss = 1.558
+Epoch 13: loss = 1.320
+Epoch 14: loss = 0.911
+Epoch 15: loss = 0.808
+Epoch 16: loss = 0.932
+Epoch 17: loss = 0.861
+Epoch 18: loss = 0.748
+Epoch 19: loss = 0.783
+Epoch 20: loss = 0.809
+Epoch 20: validation accuracy = 65.5%
+Epoch 21: loss = 0.678
+Epoch 22: loss = 0.757
+Epoch 23: loss = 0.747
+Epoch 24: loss = 0.660
+Epoch 25: loss = 0.536
+Epoch 26: loss = 0.506
+Epoch 27: loss = 0.577
+Epoch 28: loss = 0.600
+Epoch 29: loss = 0.681
+Epoch 30: loss = 0.604
+Epoch 30: validation accuracy = 68.0%
+Epoch 31: loss = 0.552
+Epoch 32: loss = 0.671
+Epoch 33: loss = 0.604
+Epoch 34: loss = 0.600
+Epoch 35: loss = 0.818
+Epoch 36: loss = 0.659
+Epoch 37: loss = 0.375
+Epoch 38: loss = 0.380
+Epoch 39: loss = 0.418
+Epoch 40: loss = 0.431
+Epoch 40: validation accuracy = 73.5%
+Epoch 41: loss = 0.551
+Epoch 42: loss = 0.488
+Epoch 43: loss = 0.350
+Epoch 44: loss = 0.287
+Epoch 45: loss = 0.294
+Epoch 46: loss = 0.463
+Epoch 47: loss = 0.438
+Epoch 48: loss = 0.392
+Epoch 49: loss = 0.325
+Epoch 50: loss = 0.332
+Epoch 50: validation accuracy = 80.8%
+Model saved in .\saved_models\adam_optim_cuda.pth

hw1/HW1-Report/codes/self_test.in.txt

@@ -0,0 +1 @@
+python recognition.py --mode test --hsize 64 --model_path .\saved_models\adam_optim_lr1e-3_epoch100_momentum10.pth

hw1/HW1-Report/codes/self_test.out.txt

@@ -0,0 +1,2 @@
+[Info] Load model from .\saved_models\adam_optim_lr1e-3_epoch100_momentum10.pth
+[Info] Test accuracy = 88.8%

hw1/HW1-Report/codes/self_train.in.txt

@@ -0,0 +1 @@
+python recognition.py --mode train --hsize 64 --lr 1e-3 --optim_type adam --momentum 10 --weight_decay 0.1 --epoch 100 --model_path .\saved_models\adam_optim_lr1e-3_epoch100_momentum10.pth

hw1/HW1-Report/codes/self_train.out.txt

@@ -0,0 +1,111 @@
+Epoch 01: loss = inf
+Epoch 02: loss = inf
+Epoch 03: loss = inf
+Epoch 04: loss = inf
+Epoch 05: loss = inf
+Epoch 06: loss = inf
+Epoch 07: loss = inf
+Epoch 08: loss = inf
+Epoch 09: loss = inf
+Epoch 10: loss = inf
+Epoch 10: validation accuracy = 40.2%
+Epoch 11: loss = inf
+Epoch 12: loss = inf
+Epoch 13: loss = inf
+Epoch 14: loss = inf
+Epoch 15: loss = inf
+Epoch 16: loss = inf
+Epoch 17: loss = 2.360
+Epoch 18: loss = 2.086
+Epoch 19: loss = 1.684
+Epoch 20: loss = 1.453
+Epoch 20: validation accuracy = 53.0%
+Epoch 21: loss = 1.174
+Epoch 22: loss = 1.046
+Epoch 23: loss = 0.859
+Epoch 24: loss = 0.740
+Epoch 25: loss = 0.663
+Epoch 26: loss = 0.495
+Epoch 27: loss = 0.566
+Epoch 28: loss = 0.521
+Epoch 29: loss = 0.470
+Epoch 30: loss = 0.363
+Epoch 30: validation accuracy = 59.0%
+Epoch 31: loss = 0.365
+Epoch 32: loss = 0.305
+Epoch 33: loss = 0.333
+Epoch 34: loss = 0.293
+Epoch 35: loss = 0.191
+Epoch 36: loss = 0.295
+Epoch 37: loss = 0.275
+Epoch 38: loss = 0.461
+Epoch 39: loss = 0.509
+Epoch 40: loss = 0.298
+Epoch 40: validation accuracy = 65.2%
+Epoch 41: loss = 0.186
+Epoch 42: loss = 0.395
+Epoch 43: loss = 0.323
+Epoch 44: loss = 0.309
+Epoch 45: loss = 0.199
+Epoch 46: loss = 0.285
+Epoch 47: loss = 0.290
+Epoch 48: loss = 0.302
+Epoch 49: loss = 0.235
+Epoch 50: loss = 0.190
+Epoch 50: validation accuracy = 71.2%
+Epoch 51: loss = 0.294
+Epoch 52: loss = 0.311
+Epoch 53: loss = 0.254
+Epoch 54: loss = 0.289
+Epoch 55: loss = 0.264
+Epoch 56: loss = 0.213
+Epoch 57: loss = 0.166
+Epoch 58: loss = 0.218
+Epoch 59: loss = 0.231
+Epoch 60: loss = 0.283
+Epoch 60: validation accuracy = 74.8%
+Epoch 61: loss = 0.324
+Epoch 62: loss = 0.245
+Epoch 63: loss = 0.277
+Epoch 64: loss = 0.286
+Epoch 65: loss = 0.255
+Epoch 66: loss = 0.263
+Epoch 67: loss = 0.272
+Epoch 68: loss = 0.272
+Epoch 69: loss = 0.260
+Epoch 70: loss = 0.271
+Epoch 70: validation accuracy = 79.0%
+Epoch 71: loss = 0.310
+Epoch 72: loss = 0.301
+Epoch 73: loss = 0.305
+Epoch 74: loss = 0.311
+Epoch 75: loss = 0.329
+Epoch 76: loss = 0.295
+Epoch 77: loss = 0.300
+Epoch 78: loss = 0.316
+Epoch 79: loss = 0.326
+Epoch 80: loss = 0.352
+Epoch 80: validation accuracy = 77.5%
+Epoch 81: loss = 0.344
+Epoch 82: loss = 0.326
+Epoch 83: loss = 0.326
+Epoch 84: loss = 0.335
+Epoch 85: loss = 0.342
+Epoch 86: loss = 0.361
+Epoch 87: loss = 0.337
+Epoch 88: loss = 0.339
+Epoch 89: loss = 0.339
+Epoch 90: loss = 0.341
+Epoch 90: validation accuracy = 82.8%
+Epoch 91: loss = 0.350
+Epoch 92: loss = 0.359
+Epoch 93: loss = 0.352
+Epoch 94: loss = 0.363
+Epoch 95: loss = 0.347
+Epoch 96: loss = 0.341
+Epoch 97: loss = 0.336
+Epoch 98: loss = 0.348
+Epoch 99: loss = 0.365
+Epoch 100: loss = 0.350
+Epoch 100: validation accuracy = 85.2%
+Model saved in .\saved_models\adam_optim_lr1e-3_epoch100_momentum10.pth

hw1/HW1-Report/dtx-style.sty

@@ -0,0 +1,132 @@
+%%
+%% This is file `dtx-style.sty',
+%% generated with the docstrip utility.
+%%
+%% The original source files were:
+%%
+%% thucoursework.dtx  (with options: `dtx-style')
+%% 
+%% This is a generated file.
+%% 
+%% Copyright (C) 2021 by zhaofeng-shu33 <616545598@qq.com>
+%% 
+%% This work may be distributed and/or modified under the
+%% conditions of the LaTeX Project Public License, either version 1.3
+%% of this license or (at your option) any later version.
+%% The latest version of this license is in
+%%   http://www.latex-project.org/lppl.txt
+%% and version 1.3 or later is part of all distributions of LaTeX
+%% version 2005/12/01 or later.
+%% 
+%% To produce the documentation run the original source files ending with `.dtx'
+%% through LaTeX.
+%% 
+
+\ProvidesPackage{dtx-style}
+\RequirePackage{hypdoc}
+\RequirePackage[UTF8,scheme=chinese]{ctex}
+\RequirePackage{newpxtext}
+\RequirePackage{newpxmath}
+\RequirePackage[
+  top=2.5cm, bottom=2.5cm,
+  left=4cm, right=2cm,
+  headsep=3mm]{geometry}
+\RequirePackage{array,longtable,booktabs}
+\RequirePackage{listings}
+\RequirePackage{fancyhdr}
+\RequirePackage{xcolor}
+\RequirePackage{enumitem}
+\RequirePackage{etoolbox}
+\RequirePackage{metalogo}
+
+\colorlet{thu@macro}{blue!60!black}
+\colorlet{thu@env}{blue!70!black}
+\colorlet{thu@option}{purple}
+\patchcmd{\PrintMacroName}{\MacroFont}{\MacroFont\bfseries\color{thu@macro}}{}{}
+\patchcmd{\PrintDescribeMacro}{\MacroFont}{\MacroFont\bfseries\color{thu@macro}}{}{}
+\patchcmd{\PrintDescribeEnv}{\MacroFont}{\MacroFont\bfseries\color{thu@env}}{}{}
+\patchcmd{\PrintEnvName}{\MacroFont}{\MacroFont\bfseries\color{thu@env}}{}{}
+
+\def\DescribeOption{%
+  \leavevmode\@bsphack\begingroup\MakePrivateLetters%
+  \Describe@Option}
+\def\Describe@Option#1{\endgroup
+  \marginpar{\raggedleft\PrintDescribeOption{#1}}%
+  \thu@special@index{option}{#1}\@esphack\ignorespaces}
+\def\PrintDescribeOption#1{\strut \MacroFont\bfseries\sffamily\color{thu@option} #1\ }
+\def\thu@special@index#1#2{\@bsphack
+  \begingroup
+    \HD@target
+    \let\HDorg@encapchar\encapchar
+    \edef\encapchar usage{%
+      \HDorg@encapchar hdclindex{\the\c@HD@hypercount}{usage}%
+    }%
+    \index{#2\actualchar{\string\ttfamily\space#2}
+           (#1)\encapchar usage}%
+    \index{#1:\levelchar#2\actualchar
+           {\string\ttfamily\space#2}\encapchar usage}%
+  \endgroup
+  \@esphack}
+
+\lstdefinestyle{lstStyleBase}{%
+   basicstyle=\small\ttfamily,
+   aboveskip=\medskipamount,
+   belowskip=\medskipamount,
+   lineskip=0pt,
+   boxpos=c,
+   showlines=false,
+   extendedchars=true,
+   upquote=true,
+   tabsize=2,
+   showtabs=false,
+   showspaces=false,
+   showstringspaces=false,
+   numbers=none,
+   linewidth=\linewidth,
+   xleftmargin=4pt,
+   xrightmargin=0pt,
+   resetmargins=false,
+   breaklines=true,
+   breakatwhitespace=false,
+   breakindent=0pt,
+   breakautoindent=true,
+   columns=flexible,
+   keepspaces=true,
+   gobble=2,
+   framesep=3pt,
+   rulesep=1pt,
+   framerule=1pt,
+   backgroundcolor=\color{gray!5},
+   stringstyle=\color{green!40!black!100},
+   keywordstyle=\bfseries\color{blue!50!black},
+   commentstyle=\slshape\color{black!60}}
+
+\lstdefinestyle{lstStyleShell}{%
+   style=lstStyleBase,
+   frame=l,
+   rulecolor=\color{purple},
+   language=bash}
+
+\lstdefinestyle{lstStyleLaTeX}{%
+   style=lstStyleBase,
+   frame=l,
+   rulecolor=\color{violet},
+   language=[LaTeX]TeX}
+
+\lstnewenvironment{latex}{\lstset{style=lstStyleLaTeX}}{}
+\lstnewenvironment{shell}{\lstset{style=lstStyleShell}}{}
+
+\setlist{nosep}
+
+\DeclareDocumentCommand{\option}{m}{\textsf{#1}}
+\DeclareDocumentCommand{\env}{m}{\texttt{#1}}
+\DeclareDocumentCommand{\pkg}{s m}{%
+  \texttt{#2}\IfBooleanF#1{\thu@special@index{package}{#2}}}
+\DeclareDocumentCommand{\file}{s m}{%
+  \texttt{#2}\IfBooleanF#1{\thu@special@index{file}{#2}}}
+\newcommand{\myentry}[1]{%
+  \marginpar{\raggedleft\color{purple}\bfseries\strut #1}}
+\newcommand{\note}[2][Note]{{%
+  \color{magenta}{\bfseries #1}\emph{#2}}}
+
+\def\thucoursework{\textsc{Thu}\-\textsc{Coursework}}

hw1/HW1-Report/iidef.sty

@@ -0,0 +1,153 @@
+%%
+%% This is file `iidef.sty',
+%% generated with the docstrip utility.
+%%
+%% The original source files were:
+%%
+%% thucoursework.dtx  (with options: `sty')
+%% 
+%% This is a generated file.
+%% 
+%% Copyright (C) 2021 by zhaofeng-shu33 <616545598@qq.com>
+%% 
+%% This work may be distributed and/or modified under the
+%% conditions of the LaTeX Project Public License, either version 1.3
+%% of this license or (at your option) any later version.
+%% The latest version of this license is in
+%%   http://www.latex-project.org/lppl.txt
+%% and version 1.3 or later is part of all distributions of LaTeX
+%% version 2005/12/01 or later.
+%% 
+%% To produce the documentation run the original source files ending with `.dtx'
+%% through LaTeX.
+%% 
+
+\NeedsTeXFormat{LaTeX2e}[1999/12/01]
+\ProvidesClass{iidef}
+[2020/09/09 2.6 Tsinghua University Coursework Template]
+%% configuration of nested enumerate env
+\RequirePackage{enumitem}
+%% set hwcount key-value option
+\RequirePackage{kvoptions}
+%% required by macro DeclareMathOperator
+\RequirePackage{amsmath}
+%% Set up page headers using with fancyhdr
+\@ifundefined{lhead}{\RequirePackage{fancyhdr}}
+{\def\@thulhead{thulhead}}
+\RequirePackage{amsthm}
+%% semester
+\def\@term{term}
+\newcommand{\theterm}[1]{\renewcommand\@term{#1}}
+%% institute
+\newcommand{\@courseinstitute}[1]{institute}
+\newcommand{\thecourseinstitute}[1]{\renewcommand\@courseinstitute{#1}}
+%% coursename
+\newcommand{\@coursename}[1]{coursename}
+\newcommand{\thecoursename}[1]{\renewcommand\@coursename{\textsc{#1}}}
+%% user can rewrite homework name
+\def\@hwname{Homework}
+\def\hwname#1{\renewcommand\@hwname{#1}}
+%% \iidef@thehwcnt = 1
+\DeclareStringOption[1]{thehwcnt}
+\ProcessKeyvalOptions*
+\def\thehwcnt{\iidef@thehwcnt}
+%% page header setup, distinguish between first page(plain style)
+%% and second page on (runningpage style)
+%%***************************************************************************
+\newcommand{\courseheader}{
+\thispagestyle{plain}%first page use native plain style to suppress header
+\vspace*{-1in}
+\begin{center}
+\@courseinstitute\\
+\@coursename\\
+\@term
+\vspace*{0.1in}
+\hrule
+\end{center}
+\begin{center}
+  \underline{\bf \@hwname\;\thehwcnt} \\
+\end{center}
+}
+\@ifundefined{@thulhead}{
+\fancypagestyle{runningpage}
+{
+  \fancyhead[L]{\small\@coursename}
+  \fancyhead[R]{\small\@courseinstitute}
+}
+%% use runningpage style from second page on
+\pagestyle{runningpage}
+}{}
+%% *********************************************************************************************
+%%name command macro
+%%*************************
+\newcommand{\name}[1]{
+\begin{flushleft}
+  #1\hfill
+  \today
+\end{flushleft}
+\hrule
+
+\vspace{2em}
+
+\flushleft
+}
+%%*************************
+%% enumitem related configuration
+\setlist[enumerate,1]{label=\thehwcnt.\arabic*.}
+\setlist[enumerate,2]{label=(\alph*)}
+\setlist[enumerate,3]{label=\roman*.}
+\setlist[enumerate,4]{label=\greek*}
+%%******************************
+\def\@slname{Solution}
+\def\slname#1{\renewcommand\@slname{#1}}
+
+\@ifundefined{solution}{
+\newenvironment{solution}
+{
+\proof[\@slname]
+}
+{
+%% no qed symbol in solution env
+\renewcommand{\qedsymbol}{}
+\endproof
+}
+}{}
+%%******************************
+%%common math symbols go here
+%%*************************************************
+\def\v#1{\underline{#1}}
+\newcommand{\uc}{\underline{c}}    % c, vec
+\newcommand{\uv}{\underline{v}}    % x, vec
+\newcommand{\uw}{\underline{w}}    % w, vec
+\newcommand{\ux}{\underline{x}}    % x, vec
+\newcommand{\uy}{\underline{y}}    % y, vec
+\newcommand{\uz}{\underline{z}}    % z, vec
+\newcommand{\um}{\underline{m}}    % m, vec
+\newcommand{\rvx}{\mathsf{x}}    % x, r.v.
+\newcommand{\rvy}{\mathsf{y}}    % y, r.v.
+\newcommand{\rvz}{\mathsf{z}}    % z, r.v.
+\newcommand{\rvw}{\mathsf{w}}    % w, r.v.
+\newcommand{\rvH}{\mathsf{H}}    % H, r.v.
+\newcommand{\urvx}{\underline{\mathsf{x}}}    % x, r.v. vec
+\newcommand{\urvy}{\underline{\mathsf{y}}}    % y, r.v. vec
+\newcommand{\urvz}{\underline{\mathsf{z}}}    % z, r.v. vec
+\newcommand{\urvw}{\underline{\mathsf{w}}}    % w, r.v. vec
+
+\newcommand{\defas}{\triangleq} %\coloneqq
+\newcommand{\reals}{\mathbb{R}}
+\newcommand{\TT}{\mathrm{T}}    % transpose
+\DeclareMathOperator*{\argmax}{arg\,max}
+\DeclareMathOperator*{\argmin}{arg\,min}
+\DeclareMathOperator*{\argsup}{arg\,sup}
+\DeclareMathOperator*{\arginf}{arg\,inf}
+\DeclareMathOperator{\diag}{diag}
+\DeclareMathOperator{\Var}{Var}
+\DeclareMathOperator{\Cov}{Cov}
+\DeclareMathOperator{\MSE}{MSE}
+\DeclareMathOperator{\1}{\mathds{1}}
+\DeclareMathOperator{\In}{\mathbb{I}}
+\DeclareMathOperator{\E}{\mathbb{E}}
+\DeclareMathOperator{\Prob}{\mathbb{P}}
+\newcommand\independent{\protect\mathpalette{\protect\independenT}{\perp}}
+\def\independenT#1#2{\mathrel{\rlap{$#1#2$}\mkern2mu{#1#2}}}
+%%************************************************************************************

hw1/HW1-Report/main.tex

@@ -0,0 +1,236 @@
+% Homework Template
+\documentclass[a4paper]{article}
+\usepackage{ctex}
+\usepackage{amsmath, amssymb, amsthm}
+\usepackage{moreenum}
+\usepackage{mathtools}
+\usepackage{url}
+\usepackage{bm}
+\usepackage{enumitem}
+\usepackage{graphicx}
+\usepackage{subcaption}
+\usepackage{booktabs} % toprule
+\usepackage[mathcal]{eucal}
+\usepackage[thehwcnt = 1]{iidef}
+\usepackage{listings}
+\usepackage[x11names]{xcolor}
+\usepackage{float}
+
+\DeclareMathOperator{\arctanh}{arctanh}
+% \DeclareMathOperator{\diag}{diag}
+
+\setenumerate[1]{label=(\arabic{*})}
+\setenumerate[2]{label=\arabic{*})}
+
+\definecolor{codekeyword}{RGB}{171, 0, 216}
+\definecolor{codetypename}{RGB}{29, 37, 251}
+\definecolor{codevariable}{RGB}{10, 23, 126}
+\definecolor{codestring}{RGB}{157, 0, 25}
+\definecolor{codecomment}{RGB}{31, 129, 19}
+
+\newfontfamily\cascadia[Ligatures=ResetAll]{Cascadia Code}
+% \newfontfamily\codefont[Ligatures=ResetAll]{Cascadia Code}
+\newfontfamily\codefont[Ligatures=ResetAll]{Fira Code}[Contextuals={Alternate}]
+% To enable ligature in listing, go check lstfiracode's github page and copy firacodestyle's settings.
+
+\lstset{
+    basicstyle          =   \small\codefont,
+    % ---
+    tabsize             =   4,
+    showstringspaces    =   false,
+    numbers             =   left,
+    numberstyle         =   \cascadia,
+    % ---
+    breaklines          =   true,
+    captionpos          =   t,      
+    % ---
+    frame               =   l,
+    flexiblecolumns,
+    columns = fixed,
+}
+
+\thecourseinstitute{清华大学电子工程系}
+\thecoursename{\textbf{媒体与认知} \space 课堂2}
+\theterm{2023-2024学年春季学期}
+\hwname{作业}
+\begin{document}
+\courseheader
+% 请在YOUR NAME处填写自己的姓名
+\name{高艺轩}
+\vspace{3mm}
+\centerline{\textbf{\Large{理论部分}}}
+
+\section{单选题（15分）}
+% 请在？处填写答案
+\subsection{\underline{B}}
+
+\subsection{\underline{A}}
+
+\subsection{\underline{B}}
+
+\subsection{\underline{A}}
+
+\subsection{\underline{B}}
+
+\section{计算题（15 分）}
+\subsection{设隐含层为$\mathbf{z}=\mathbf{W}^T\mathbf{x}+\mathbf{b}$，其中$\mathbf{x}\in R^{(m \times 1)}$，$\mathbf{z}\in R^{(n\times 1)}$，$\mathbf{W}\in R^{(m\times n)}$，$\mathbf{b} \in R^{(n\times 1)}$均为已知，其激活函数如下：
+$$\mathbf{y}=\delta(\mathbf{z})=tanh(\mathbf{z})$$
+tanh表示双曲正切函数。若训练过程中的目标函数为L，且已知L对$\mathbf{y}$的导数 $\frac{\partial L}{\partial \mathbf{y}}=[\frac{\partial L}{\partial y_1},\frac{\partial L}{\partial y_2},...,\frac{\partial L}{\partial y_n}]^T$和$\mathbf{y}=[y_1,y_2,...,y_n]^T$的值。
+}
+\subsubsection{请使用$\mathbf{y}$表示出$\frac{\partial \mathbf{y}^T}{\partial \mathbf{z}}$, 这里的$\mathbf{y}^T$ 为行向量。
+}
+
+\begin{proof}[解]
+    首先，对$i \neq j$，$\dfrac{\partial y_i}{\partial z_j} = 0$。
+    
+    同时$y_i = \tanh(z_i) = \tanh(\arctanh(y_i))$，因此
+    \[\frac{\partial y_i}{\partial z_i} = 1 - \tanh^2(z_i) = 1 - y_i^2\]
+    因此
+    \[\dfrac{\partial \boldsymbol{y}^\mathrm{T}}{\partial \boldsymbol{z}} = \diag\{1 - y_1^2, 1 - y_2^2, \dots, 1 - y_n^2\} \qedhere\]
+\end{proof}
+
+\subsubsection{请使用$\mathbf{y}$和$\frac{\partial L}{\partial \mathbf{y}}$表示$\frac{\partial L}{\partial \mathbf{x}}$，$\frac{\partial L}{\partial \mathbf{W}}$，$\frac{\partial L}{\partial \mathbf{b}}$。
+}
+提示:$\frac{\partial L}{\partial \mathbf{x}}$，$\frac{\partial L}{\partial \mathbf{W}}$，$\frac{\partial L}{\partial \mathbf{b}}$与x,W,b具有相同维度。
+
+\begin{proof}[解]
+    由链式法则
+    \[\frac{\partial L}{\partial \boldsymbol{x}} = \frac{\partial \boldsymbol{z}^\mathrm{T}}{\partial \boldsymbol{x}} \frac{\partial \boldsymbol{y}^\mathrm{T}}{\partial \boldsymbol{z}} \frac{\partial L}{\partial \boldsymbol{y}} = W \diag\{1 - y_1^2, 1 - y_2^2, \dots, 1 - y_n^2\} \frac{\partial L}{\partial \boldsymbol{y}}\]
+
+    对于$\dfrac{\partial L}{\partial W}$，
+    \[\frac{\partial \boldsymbol{z}^T}{\partial W} = \begin{bmatrix}
+        \boldsymbol{x} & \boldsymbol{x} & \cdots & \boldsymbol{x}
+    \end{bmatrix}_{m \times n}\]
+
+    \begin{align*}
+        \frac{\partial L}{\partial W} & = \frac{\partial \boldsymbol{z}^\mathrm{T}}{\partial W} \frac{\partial \boldsymbol{y}^\mathrm{T}}{\partial \boldsymbol{z}} \frac{\partial L}{\partial \boldsymbol{y}}\\
+        & = \begin{bmatrix}
+        \boldsymbol{x} & \boldsymbol{x} & \cdots & \boldsymbol{x}
+    \end{bmatrix}_{m \times n} \diag\{1 - y_1^2, 1 - y_2^2, \dots, 1 - y_n^2\} \frac{\partial L}{\partial \boldsymbol{y}}
+    \end{align*}
+
+    对于$\dfrac{\partial L}{\partial \boldsymbol{b}}$，由链式法则
+    \[\frac{\partial L}{\partial \boldsymbol{b}} = \frac{\partial \boldsymbol{z}^\mathrm{T}}{\partial \boldsymbol{b}} \frac{\partial \boldsymbol{y}^\mathrm{T}}{\partial \boldsymbol{z}} \frac{\partial L}{\partial \boldsymbol{y}} = I_n \frac{\partial \boldsymbol{y}^\mathrm{T}}{\partial \boldsymbol{z}} \frac{\partial L}{\partial \boldsymbol{y}} = \diag\{1 - y_1^2, 1 - y_2^2, \dots, 1 - y_n^2\} \frac{\partial L}{\partial \boldsymbol{y}} \qedhere\]
+\end{proof}
+\vspace{6mm}
+\centerline{\textbf{\Large{编程部分}}}
+
+
+\vspace{3mm}
+% 请根据是否选择自选课题的情况选择“编程作业报告”或“自选课题开题报告”中的一项完成
+\section{编程作业报告}
+% 请在此处完成编程作业报告
+\begin{enumerate}
+    \item 使用默认配置进行训练和测试。
+    \begin{enumerate}
+        \item 训练模型。
+        
+        输入：
+        \lstinputlisting{codes/1.1.in.txt}
+        
+        输出：
+        \lstinputlisting{codes/1.1.out.txt}
+        \begin{figure}[H]
+            \centering
+            \includegraphics[width=0.9\linewidth]{img/1default_train.png}
+        \end{figure}
+
+        \item 测试模型。
+        
+        输入：
+        \lstinputlisting{codes/1.2.in.txt}
+
+        输出：
+        \lstinputlisting{codes/1.2.out.txt}
+    \end{enumerate}
+    \item 调整参数、使用Adam优化器训练并测试。
+    \begin{enumerate}
+        \item 训练模型。
+        
+        输入：
+        \lstinputlisting{codes/2.1.in.txt}
+
+        输出：
+        \lstinputlisting{codes/2.1.out.txt}
+        \begin{figure}[H]
+            \centering
+            \includegraphics[width=0.9\linewidth]{img/2adam_optim.png}
+        \end{figure}
+        \item 测试性能。
+        
+        输入：
+        \lstinputlisting{codes/2.2.in.txt}
+
+        输出：
+        \lstinputlisting{codes/2.2.out.txt}
+    \end{enumerate}
+
+    \item 使用效果最佳的模型测试。
+    经过简单的尝试，发现使用
+    \lstinputlisting{codes/self_train.in.txt}
+    可以使测试集准确率达到88.8\%，有略微的提升。训练的loss曲线：
+    \begin{figure}[H]
+        \centering
+        \includegraphics[width=.9\linewidth]{img/3found_best.png}
+    \end{figure}
+    使用它进行预测：
+    \begin{figure}[H]
+        \centering
+        \begin{subfigure}[b]{.3\linewidth}
+            \includegraphics[width=\linewidth]{img/predict/predict01.png}
+            \subcaption{预测：A}
+        \end{subfigure}
+        \hfill
+        \begin{subfigure}[b]{.3\linewidth}
+            \includegraphics[width=\linewidth]{img/predict/predict02.png}
+            \subcaption{预测：B}
+        \end{subfigure}
+        \hfill
+        \begin{subfigure}[b]{.3\linewidth}
+            \includegraphics[width=\linewidth]{img/predict/predict03.png}
+            \subcaption{预测：M}
+        \end{subfigure}
+
+        \begin{subfigure}[b]{.3\linewidth}
+            \includegraphics[width=\linewidth]{img/predict/predict04.png}
+            \subcaption{预测：R}
+        \end{subfigure}
+        \hfill
+        \begin{subfigure}[b]{.3\linewidth}
+            \includegraphics[width=\linewidth]{img/predict/predict05.png}
+            \subcaption{预测：M}
+        \end{subfigure}
+        \hfill
+        \begin{subfigure}[b]{.3\linewidth}
+            \includegraphics[width=\linewidth]{img/predict/predict06.png}
+            \subcaption{预测：O}
+        \end{subfigure}
+
+        \hfill
+        \begin{subfigure}[b]{.3\linewidth}
+            \includegraphics[width=\linewidth]{img/predict/predict07.png}
+            \subcaption{预测：B}
+        \end{subfigure}
+        \hfill
+        \begin{subfigure}[b]{.3\linewidth}
+            \includegraphics[width=\linewidth]{img/predict/predict08.png}
+            \subcaption{预测：W}
+        \end{subfigure}
+        \hfill
+    \end{figure}
+\end{enumerate}
+
+
+
+
+% \section{自选课题开题报告}
+% 请在此处介绍自选课题
+
+\end{document}
+
+
+
+%%% Local Variables:
+%%% mode: late\rvx
+%%% TeX-master: t
+%%% End:

hw1/HW1-code/activations.py

@@ -0,0 +1,164 @@
+#========================================================
+#             Media and Cognition
+#             Homework 1 Neural network basics
+#             activations.py - activation functions
+#             Student ID: 2022010639
+#             Name: Gao Yixuan
+#             Tsinghua University
+#             (C) Copyright 2024
+#========================================================
+import torch
+import torch.nn as nn
+
+'''
+In this script we will implement three activation functions, including both forward and backward processes.
+More details about customizing a backward process in PyTorch can be found in:
+https://pytorch.org/tutorials/beginner/examples_autograd/two_layer_net_custom_function.html
+'''
+
+## Here, Tanh is given as an example to show how to construct the activation function. Please finish the activation functions of Sigmoid and ReLU later.
+class Tanh(torch.autograd.Function):
+    '''
+    Tanh activation function
+    y = (exp(x) - exp(-x)) / (exp(x) + exp(-x))
+    '''
+    # static method of a python class means that we can call the function without initializing an instance of the class
+    @staticmethod
+    def forward(ctx, x):
+        '''
+        In the forward pass we receive a Tensor containing the input x and return
+        a Tensor containing the output. 
+        
+        ctx: it is a context object that can be used to save information for backward computation. You can save 
+        objects by using ctx.save_for_backward, and get objects by using ctx.saved_tensors
+
+        x: input with arbitrary shape
+        '''
+        # Please think if we use "y = (exp(x) - exp(-x)) / (exp(x) + exp(-x))", what might happen when x has a large absolute value
+        # y = (torch.exp(x) - torch.exp(-x)) / (torch.exp(x) + torch.exp(-x))
+
+        # here we directly use torch.tanh(x) to avoid the problem above
+        y = torch.tanh(x)
+
+        # save an variable in ctx
+        ctx.save_for_backward(y)
+
+        return y
+
+    @staticmethod
+    def backward(ctx, grad_output):
+        """
+        In the backward pass we receive a Tensor containing the gradient of the loss
+        with respect to the output, and we need to compute the gradient of the loss
+        with respect to the input.
+
+        grad_output: dL/dy
+        grad_input: dL/dx = dL/dy * dy/dx, where y = forward(x)
+        """
+        # get an variable from ctx
+        y, = ctx.saved_tensors
+
+        # chain rule: dL/dx = dL/dy * dy/dx
+        # where dL/dy = grad_output, and the dy/dx of tanh function is (1-y^2)!
+        grad_input = grad_output * (1 - y ** 2)
+
+        return grad_input
+
+#TODO 1: complete the forward and backward functions of the Sigmoid activation function.
+#Note: You can refer to the activation function Tanh
+class Sigmoid(torch.autograd.Function):
+    '''
+    Sigmoid activation function
+    y = 1 / (1 + exp(-x))
+    '''
+
+    @staticmethod
+    def forward(ctx, x):
+
+        # hint: you can use torch.exp(x) to calculate exp(x)
+        y = 1 - (1 + torch.exp(-x))
+
+        # here we save y in ctx, in this way we can use y to calculate gradients in backward process
+        ctx.save_for_backward(y)
+
+        return y
+
+    @staticmethod
+    def backward(ctx, grad_output):
+
+        # get y from ctx
+        y, = ctx.saved_tensors
+
+        # implement gradient of x (grad_input), grad_input refers to dL/dx
+        # chain rule: dL/dx = dL/dy * dy/dx
+        # where dL/dy = grad_output, and dy/dx of Sigmoid function is y * (1 - y)
+        grad_input = grad_output * y * (1 - y)
+
+        return grad_input
+
+#TODO 2: complete the forward and backward functions of the ReLU activation function.
+#Note: You can refer to the activation function Tanh
+class ReLU(torch.autograd.Function):
+    '''
+    ReLU activation function
+    y = max{x, 0}
+    '''
+
+    @staticmethod
+    def forward(ctx, x):
+
+        # set elements less than 0 in x to 0
+        # this operation is inplace
+        x = torch.max(x, torch.tensor([0.]).to(x.device))
+
+        # save x in ctx, in this way we can use x to calculate gradients in backward process
+        ctx.save_for_backward(x)
+
+        # return the output
+        return x
+
+    @staticmethod
+    def backward(ctx, grad_output):
+        """
+        In the backward pass we receive a Tensor containing the gradient of the loss
+        with respect to the output, and we need to compute the gradient of the loss
+        with respect to the input.
+        """
+
+        # get x from ctx
+        x, = ctx.saved_tensors
+        # print("Before heaviside")
+        # print(x, x.size())
+        x = torch.heaviside(x, torch.tensor([0.]).to(x.device))
+        # print("After heaviside")
+        # print(x, x.size())
+        # print(grad_output, grad_output.size())
+        # print(grad_output * x)
+
+        # chain rule: dL/dx = dL/dy * dy/dx
+        # where dL/dy = grad_output, and dy/dx of ReLU function is 1 if x > 0, and 0 if x <= 0
+        grad_input = grad_output * x
+        
+        return grad_input
+
+
+# activate function class according to the type
+class Activation(nn.Module):
+    def __init__(self, type):
+        '''
+        :param type:  'sigmoid', 'tanh', or 'relu'
+        '''
+        super().__init__()
+
+        if type == 'sigmoid':
+            self.act = Sigmoid.apply
+        elif type == 'tanh':
+            self.act = Tanh.apply
+        elif type == 'relu':
+            self.act = ReLU.apply
+        else:
+            print('activation type should be one of [sigmoid, tanh, relu]')
+            raise NotImplementedError
+
+    def forward(self, x):
+        return self.act(x)