\documentclass{bmvc2k}
\usepackage[nolist,nohyperlinks]{acronym}
\usepackage{booktabs}

%% Enter your paper number here for the review copy
% \bmvcreviewcopy{2}

\title{Energy Savings Playing the Lottery}

% Enter the paper's authors in order
\addauthor{Marco Schnell}{marco.schnell@fau.de}{1}
\addauthor{Nora Gourmelon}{nora.gourmelon@fau.de}{1}
\addauthor{Vincent Christlein}{vincent.christlein@fau.de}{1}
\addauthor{Andreas Maier}{andreas.maier@fau.de}{1}
\addauthor{Mathias Seuret}{mathias.seuret@fau.de}{1}

% Enter the institutions
% \addinstitution{Name\\Address}
\addinstitution{
Pattern Recognition Lab,
Friedrich-Alexander-Universit\"at Erlangen-N\"urnberg, 
9058 Erlangen, Germany
}

\runninghead{Schnell et al.}{Energy Savings Playing the Lottery}

% Any macro definitions you would like to include
% These are not defined in the style file, because they don't begin
% with \bmva, so they might conflict with the user's own macros.
% The \bmvaOneDot macro adds a full stop unless there is one in the
% text already.
\def\eg{\emph{e.g}\bmvaOneDot}
\def\Eg{\emph{E.g}\bmvaOneDot}
\def\etal{\emph{et al}\bmvaOneDot}

%-------------------------------------------------------------------------
% Document starts here
\begin{document}
\begin{acronym}
\acro{nn}[NN]{Neural Network}
\acro{ann}[ANN]{Artificial Neural Network}
\acro{cnn}[CNN]{Convolutional Neural Network}
\acro{resnet}[ResNet]{Residual Neural Network}
\acro{gpu}[GPU]{Graphics Processing Unit}
\acro{lth}[LTH]{Lottery Ticket Hypothesis}
\acro{nlp}[NLP]{Natural Language Processing}
\acro{snn}[SNN]{Spike Neural Network}
\acro{dnn}[DNN]{Deep Neural Network}
\acro{apoz}[APoZ]{Average Percentage of Zeros}
\acro{cel}[CE]{Cross-Entropy}
\acro{adam}[ADAM]{Adaptive Moment Estimation}
\acro{relu}[ReLU]{Rectified Linear Unit}
\acro{eb}[EB]{Early-Bird}
\acro{nas}[NAS]{Neural Architecture Search}
\end{acronym}

\maketitle

\begin{abstract}
The rapid development of modern neural networks has led to highly over-parameterized models, resulting in excessive memory usage, computation, and energy consumption at inference time. 
In this paper, we propose a structured pruning method inspired by the Lottery Ticket Hypothesis, aiming to reduce the network size while preserving accuracy. 
Our method removes entire neurons based on a magnitude-based selection criterion~--~unlike the conventional unstructured approach of setting weights to zero via the utilization of binary masks. 
Hereby, we demonstrate the efficacy of magnitude-based layer and neuron selection techniques that guide our structured pruning algorithm without the necessity of complex search patterns.
We validate our method on two distinct scenarios: the well known CIFAR-100 dataset, and a document image analysis task. 
We evaluate the benefits of our methodology using GPU-based energy measurements and show that our pruned networks can reduce the energy consumption per sample by more than 40\,\% [Wh/sample], with comparable or slightly superior test accuracies. 
These findings highlight the potential of structured pruning to create energy-efficient neural networks suitable for deployment in resource-constrained environments.
\end{abstract}

%-------------------------------------------------------------------------


\section{Introduction}

\acp{ann} have demonstrated significant success in real-world applications such as object recognition, image classification, autonomous driving, and natural language processing~\citep{Vadera22:MethodsPruningDNN, Blalock20:StateNNPruning, Liang21:PruningQuantizationDNNSurvey, Ko19:MUP}.
Recent advancements, such as OpenAI’s ChatGPT, have increased public interest in the possibilities enabled by \acp{ann}~\citep{Yang24:LLMsSurvey}.
However, the growing adoption of such models is accompanied by significantly higher energy requirements.

A core issue behind this inefficiency is the large number of parameters in state-of-the-art \acp{ann}, which lead to impressive accuracy but also to substantial memory and computational resource demands~\citep{Ko19:MUP, Vadera22:MethodsPruningDNN, SeulKi21:TowardsCompactDNNvEAPruning, Blalock20:StateNNPruning}.
Consequently, energy consumption increased, particularly in over-parameterized \acp{dnn}~\citep{You19:DrawingEBTicketsEfficientTraining, Yang16:DEECNNuEAP, Liang21:PruningQuantizationDNNSurvey, Rachwan22:WinningLTHAhead, Yang24:LLMsSurvey}.
Considering that the energy sector contributes approximately 34\,\% to global greenhouse gas emissions, there is a pressing need for more energy-efficient \acp{ann}~\citep{Lee23:IPCC}.

Among various compression techniques, knowledge distillation~\citep{Hinton15:KD} and pruning~\citep{LeCun89:OBD, Han15:LbWCfENN, Frankle19:LTH, Blalock20:StateNNPruning, Liang21:PruningQuantizationDNNSurvey, SeulKi21:TowardsCompactDNNvEAPruning, Gerum2020:SparsitytEvolPruning} stand out.
However, knowledge distillation may not sufficiently eliminate redundancy and achieve significant compression~\citep{Cui21:JointSPruningandKD, Aghli21:CombiningPruningandKD}, making pruning a more promising avenue for compression. The \ac{lth}, introduced by~\citet{Frankle19:LTH}, demonstrated that sparse sub-networks derived from a larger network can achieve similar or better performance.
While \ac{lth} traditionally applies unstructured pruning (i.\,e., setting weights to zero), our work instead explores the effect of completely removing parameters via structured pruning~\citep{Rachwan22:WinningLTHAhead}.
Moreover, the focus of this study is the measured reduction in energy consumption resulting from the implementation of structured pruning techniques.
In addition, we focus on convolutional layers, known to be energy-intensive~\citep{Yang16:DEECNNuEAP}, and apply a novel combination of structured pruning and \ac{lth} principles.

% Our method uses an iterative pruning scheme, guided by a straightforward magnitude-based criterion (\textit{lowest mean weights}), to remove entire neurons.
Our methodology integrates the iterative pruning process with reinitialization of weights from the original network, as proposed by the \ac{lth}, with the concept of structured pruning to remove neurons, as outlined by~\citet{Hu16:NTDDNPAtEDA}, without necessitating the implementation of sophisticated layer and neuron selection strategies. 
Instead, we employ a straightforward approach of searching for the \textit{lowest mean weights} between neurons, as these neurons have a limited impact on the network's output.
We show that this strategy can produce sub-networks with significantly fewer neurons while retaining~--~or even improving test accuracy. Furthermore, we validate our method using real GPU energy measurements during inference, demonstrating that energy consumption per sample can be reduced by over 40\,\% without compromising accuracy.
This reduction not only lowers the environmental footprint of \acp{nn}, but also makes them more cost-effective for practical deployment.



% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\section{Related Work}
\subsection{Pruning}
The concept of pruning dates back to the 1960s~\citep{Ivakhnenko68group, Ivakhnenko71UnitPruning}.
A major milestone was achieved by~\citet{LeCun89:OBD}, who introduced weight elimination using second-order derivatives.
Later,~\citet{Han15:LbWCfENN} proposed pruning small-magnitude weights to reduce model complexity.
Recent works have explored various pruning strategies that achieve efficiency gains with minimal performance loss~\citep{Liu17:LECNtNSLIMMING, Hu16:NTDDNPAtEDA, Molchanov17:PCNNfREI}. In particular,~\citet{Hu16:NTDDNPAtEDA} developed a structured pruning technique based on the \ac{apoz}, identifying and removing neurons with minimal activation on a validation set.
They found that reinitializing pruned networks with original weights prior to retraining helped retain accuracy, provided that pruning steps were not too aggressive.

In the context of transfer learning,~\citet{Molchanov17:PCNNfREI} introduced pruning strategies that remove entire parameters from the network.

Pruning techniques are commonly classified as either unstructured or structured~\citep{Blalock20:StateNNPruning, Renda20:ComparingRewFTiNNPLTH, Rachwan22:WinningLTHAhead}.
Unstructured pruning zeroes individual weights, which can reduce storage, but does not necessarily lead to computational gains unless the hardware supports sparse matrices~\citep{Frankle19:StabilizingLTH}.
Structured pruning, by contrast, removes neurons or entire layers, enabling real speedups and memory savings~\citep{Liu17:LECNtNSLIMMING}. Structured pruning also impacts energy consumption, which in turn affects carbon emissions~\cite{Widmann23:PruningforPowerOEEIoTNNP}. 
For instance,~\citet{Widmann23:PruningforPowerOEEIoTNNP} report energy savings of 59\,\% with only 3\,\% accuracy loss when pruning 75\,\% of a network using structured \ac{apoz} pruning.
Conversely, unstructured pruning alone showed no energy benefit unless heavily optimized~\cite{Widmann23:PruningforPowerOEEIoTNNP}. Timing of pruning also matters.
While static pruning removes neurons after training, dynamic pruning eliminates them during training~\citep{Liang21:PruningQuantizationDNNSurvey}.
% In either case, care must be taken to maintain structural integrity and functionality across layers, especially for networks with batch normalization or attention mechanisms~\citep{Hoefler21:SparsityDL:PruningaGrowthEfficientInferenceTrainingNN, Gholami22:SurveyQuantMethfENNInference}.
In the context of structured pruning, it is essential that the tensor sizes between layers are consistent, which may necessitate pruning multiple layers, e.g., in the case of batch-normalization or attention heads in transformers~\citep{Hoefler21:SparsityDL:PruningaGrowthEfficientInferenceTrainingNN}.
It is imperative to note that certain components may exhibit heightened sensitivity to pruning, potentially resulting in substantial performance implications~\citep{Gholami22:SurveyQuantMethfENNInference}.


\subsection{Lottery Ticket Hypothesis}
The \ac{lth} introduced by~\citet{Frankle19:LTH} hypothesizes that dense, over-parameterized networks contain sparse sub-networks that can be trained to match original accuracy.
The identification of these sub-networks is achieved through an iterative pruning process, resetting remaining weights to their initial (prior-training) values and retraining the network. 
This is achieved by masking zeroed connections from previous iterations.
Therefore, \ac{lth} is traditionally unstructured.Recent studies have investigated stability enhancements, alongside the acquisition of insight into the pruning mechanism and the extension of applications to domains such as \ac{nlp} and \acp{snn}~\citep{Frankle19:StabilizingLTH, Blalock20:StateNNPruning, Kim22:ExploringLTHSNN, Malach20:ProvingLTH_PruningIsAll}.A notable extension of LTH is the \ac{eb} ticket concept~\citep{You19:DrawingEBTicketsEfficientTraining}, which seeks to identify winning tickets early in training through their proposed mask distance metric and reduced precision schemes, avoiding the need for full retraining cycles.

\subsection{Quantization}
Network quantization reduces the numerical precision of weights and / or activations, e.g., from 32-bit floats to 8-bit integers~\citep{Liang21:PruningQuantizationDNNSurvey}.
This has been shown to improve memory usage, computation time, and bandwidth, while concomitantly achieving a substantial reduction in energy consumption ~\citep{Liang21:PruningQuantizationDNNSurvey,Gong14:CompressingQuantization}.
\citet{Gholami22:SurveyQuantMethfENNInference} reported that the use of 8-bit integer quantization resulted in speed enhancements of up to 5$\times$ and energy savings of up to 30$\times$. Because quantization works independently, it can be used in combination with other technologies such as pruning or knowledge distillation to optimize benefits~\citep{Liang21:PruningQuantizationDNNSurvey}.

\subsection{Neural Architecture Search}
\ac{nas} automates the design of efficient network architectures within a predefined space~\citep{Elksen19:NASSurvey}.
Unlike pruning, which reduces an existing pre-trained model, \ac{nas} constructs a new model before training from the ground up~\citep{Ding22:NAP, Elksen19:NASSurvey}.

However, \ac{nas} and pruning are not mutually exclusive~\citep{Ding22:NAP, Li19:POPSpeedAccNAS}. 
For example, \ac{nas} can be used to design an initial compact and efficient architecture, which can then be further refined through pruning to eliminate redundancies and increase computational efficiency~\citep{Ding22:NAP}.

\subsection{Energy Measurements}
There is increasing attention on the energy consumption of neural networks, considering aspects such as network type (e.g., \acp{snn} vs. \acp{dnn}) and the energy used during training, pruning, and inference~\citep{Kim22:ExploringLTHSNN}. 
Energy consumption can be assessed through estimation tools like the \textit{CodeCarbon emissions tracker}~\citep{Schmidt21:CodeCarbon, Rachwan22:WinningLTHAhead} or through real-world measurement devices~\citep{You19:DrawingEBTicketsEfficientTraining}. In a study by~\citet{Yang16:DEECNNuEAP}, hardware-based energy measurements were used to guide pruning decisions by analyzing multiply and accumulate (MAC) operations and memory access patterns of neurons or a group of weights.
We do not measure any CO$_2$ reduction or cost saving, as these values are not constant: they change from one place to another, and might change much over time.

% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\section{Method}\label{sec:method}

This paper presents a method for structured, static, and iterative pruning based on a simple yet effective neuron relevance criterion: the \textit{lowest mean weights}.
The following subsections describe our pruning method, energy measurement setup, and the overall neuron removal strategy.

% TODO Either use {figure*}[t] in environment for figures or width=\columnwidth instead of width=\textwidth
\begin{figure}[tb]
    \centering
    \includegraphics[width=.8\columnwidth]{Figures/pruning_variants_fonts_resized.pdf}
    \caption{Unstructured and structured pruning variants. 
    Inspired by Figures 4 and 5 from~\citet{Hu16:NTDDNPAtEDA}.}
    \label{fig:NetworkTrimming4a5}
\end{figure}

\begin{figure*}[tb]
    \centering
    \includegraphics[width=0.65\textwidth]{Figures/Add-Exp_ResNet50Bat_CIFAR100_NW_SubPlots_Wh_range_-0.01.pdf}
    \caption{\ac{resnet}50-Bat on CIFAR-100: test accuracy, energy usage, and throughput across network sizes.
    The x-axis on all plots shows the network size ranging from 100\,\% to 0\,\%. }
    \label{fig:ResNet50Bat_CIFAR100_NW_SubPlots_range_-0.01}
\end{figure*}

\subsection{Pruning Method}

Unlike the original \ac{lth}~\citep{Frankle19:LTH}, which uses unstructured pruning by setting weights to zero, our approach removes neurons entirely from the network.
This qualifies as structured pruning and affects both the layer being pruned and the subsequent layers. We adapt the structured pruning principle from~\citet{Hu16:NTDDNPAtEDA}, who use the \ac{apoz} metric, but simplify the process by introducing the \textit{lowest mean weight} metric to evaluate layer and neuron relevance. Previously, we investigated a method called \textit{percentile masking}, inspired by \ac{apoz}.
This technique masked neuron weights below a percentile threshold into binary values and counted zeros per neuron to determine importance.
However, the \textit{lowest mean weight} method produced more stable sub-networks with similar or improved performance, while being computationally more straightforward.

%In the \textit{lowest mean weight} method we define the mean of a neuron's weights as:
%\begin{equation}
%    mean_{l,n} = \frac{1}{k \cdot p} \sum_{c=0}^{p} \sum_{j,i=0}^{k} \| A_{l,n,c,j,i} \|,
%\end{equation}
%where $A_{l,n,c,j,i}$ is the convolution weight at layer $l$, neuron $n$, input channel $c$, and spatial location $(j,i)$.
In the \textit{lowest mean weight} method we define the mean of a neuron's weights as $mean_{l,n} = \frac{1}{k \cdot p} \sum_{c=0}^{p} \sum_{j,i=0}^{k} \| A_{l,n,c,j,i} \|$, where $A_{l,n,c,j,i}$ is the convolution weight at layer $l$, neuron $n$, input channel $c$, and spatial location $(j,i)$.

This choice is grounded in the observation that neurons with higher \textit{mean weights} often contribute more significantly to activations in subsequent layers.

The pruning process proceeds as follows in each pruning iteration.
First, we select the layer by computing the \textit{mean weights} for all layers and select the one with the overall \textit{lowest mean value}. Second, we select the neurons by ranking neurons within the selected layer based on their \textit{mean weights}. Neurons with the \textit{lowest mean weights} are considered the least important. And third, we prune the selected neuron and adjust connected layers.
We repeat this pruning process until a predefined \textit{neuron pruning percentage} is achieved or until the layer is pruned to the maximum.

The \textit{neuron pruning percentage} is introduced to establish a quantitative metric to define the degree of aggressiveness of pruning the selected layer in each pruning iteration.

\subsection{Energy Measurement}\label{sec:energymeasurement}



\subsubsection{Nvidia-smi tool}\label{subsec:nvidiasmi}
We use the \texttt{nvidia-smi} tool~\citep{NVIDIA-SMI} to log GPU power draw every second.
These values include baseline idle consumption, which we subtract from the raw readings to isolate network-related energy use. Since \texttt{nvidia-smi} outputs watt-seconds, conversion to kilowatt-hours is done using $1 Ws = \frac{1}{3600000} \text{ kWh}$. We use this tool in our neural network measurements to obtain a more accurate approximation of the actual consumption of the network and its sub-networks running exclusively on the GPU.
In a previous study, the validity of the \texttt{nvidia-smi} measurements was established through a comparison with the readings obtained from an external electricity meter.
The accuracy is $\pm 2\,\%$ compared to the external meter, but does not require manual inspections of an electricity meter.
% Further experiments validating \texttt{nvidia-smi} measurements in comparison to an electricity meter are described in the appendix~\ref{app:sec:nvidiavalidity}.

\subsubsection{Measurement Setup}\label{subsec:measetup}
Energy measurements are conducted using preloaded batches of random images on the GPU memory.
The random images have been established as equivalent in dimension to the original dataset.
Batch sizes are optimized per sub-/network in order to maximize the throughput of the GPU. The duration of each measurement is fixed at 5 minutes to ensure consistent and representative energy readings, without incurring long runtimes via \texttt{nvidia-smi} or electricity meter. Optimizing batch size for throughput is critical, as smaller pruned networks are capable of processing a larger volume of data per second on a given amount of memory. 
This results in a lower energy consumption per sample, despite a comparable total power draw.

\subsection{Neuron Removal Algorithm}
\label{enum:myAp2}
Our algorithm combines the \ac{lth} iterative pruning with structured pruning from~\citet{Hu16:NTDDNPAtEDA}, using the \textit{lowest mean weight} metric for neuron removal. 
Our objectives of attaining accuracies analogous to those of the original network are congruent with the ones of of~\cite{Hu16:NTDDNPAtEDA} and~\cite{Frankle19:LTH}.
Disparities between unstructured and structured pruning are illustrated in Fig.~\ref{fig:NetworkTrimming4a5}.


We used the same experimental process for both dataset.
We can summarize it as follows.
We start by selecting the network architecture and dataset.
Then, we train the network until the validation accuracy converges at a learning rate of 0.001.
Once this is done, we identify the layer and the specific neurons for pruning based on the \textit{lowest mean weight} method, in addition to those already pruned.
Then, if the network has more than 5\% of its initial weight count, we reset the model's weights to their \textit{initial} values, and prune selected neurons by removing output channels (weights/bias) of the selected neurons, adjusting the input channels of the subsequent layer, and updating architecture-specific layers, e.g. batch normalization and resampling layers to preserve network integrity.
We then evaluate accuracy on the test data and perform energy measurements via \texttt{nvidia-smi} on artificially generated random dataset.
We repeat this process, starting from the training phase, until over 95\,\% of the network is pruned or until all convolutional layers have only one remaining neuron.


% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\section{Experimental Setup}

% \begin{figure*}[t]
%     \centering
%     \includegraphics[width=0.75\textwidth]{Figures/ResNet50Bat_CIFAR100_NW_Gains_range_0.6.pdf}
%     \caption{\ac{resnet}50-Bat on CIFAR-100: gains in energy per sample and sample throughput.
%     The gain for each sub- and full-network is calculated by comparing its performance to that of the full network, for network sizes ranging from 100\,\% to 60\,\%.
%     \textit{Gain Energy per Sample} indicates the energy reduction used per sample.
%     \textit{Gain Samples} refers to the increase in the number of images that can be processed by the sub-networks.}
%     \label{fig:ResNet50Bat_CIFAR100_NW_Gains_range_0.6}
% \end{figure*}

\begin{figure*}[t]
    \centering
    \subfigure[\ac{resnet}50-Bat on CIFAR-100: gains in energy per sample and sample throughput.
        Gains are calculated by comparing performances to that of full networks.
        \textit{Gain Energy per Sample} indicates the energy reduction used per sample.
        \textit{Gain Samples} refers to the increase in the number of images that can be processed by the sub-networks.\label{fig:ResNet50Bat_CIFAR100_NW_Gains_range_0.6}]{
        \includegraphics[width=0.45\textwidth]{Figures/ResNet50Bat_CIFAR100_NW_Gains_range_0.6.pdf}
        % \caption{\ac{resnet}50-Bat on CIFAR-100: gains in energy per sample and sample throughput.
        % The gain for each sub- and full-network is calculated by comparing its performance to that of the full network, for network sizes ranging from 100\,\% to 60\,\%.
        % \textit{Gain Energy per Sample} indicates the energy reduction used per sample.
        % \textit{Gain Samples} refers to the increase in the number of images that can be processed by the sub-networks.}
        % \label{fig:ResNet50Bat_CIFAR100_NW_Gains_range_0.6}
    }
    \hfill
    \subfigure[\ac{resnet}50-Bat on CIFAR-100: test accuracy vs. energy per sample. 
        This plot displays the relationship between test accuracy and energy consumption per sample for all sub- and full-networks.
        The box plot at the top shows the median, lower and upper quantiles, with the whiskers representing the 1.5 inner quantile range and the dots indicating outliers in the data.\label{fig:ResNet50Bat_CIFAR100_TestAcc_Energy_range_0.6}]{
        \includegraphics[width=0.45\textwidth]{Figures/ResNet50Bat_CIFAR100_TestAcc_Energy_Wh_range_0.6.pdf}
    }
    \caption{\ac{resnet}50-Bat on CIFAR-100.}
    % \label{fig:pruning_variants}
\end{figure*}

\subsection{CIFAR-100 Dataset}\label{subsubsec:cifar}
The CIFAR-100 dataset contains 60,000 images in 100 categories.
Each image is $32 \times 32$ pixels and belongs to one of 600 images per class.
Its compact size and broad use make it ideal for benchmarking.

% \subsubsection{Fonts Dataset TODO}\label{subsubsec:fonts}
\subsection{Dataset of Pages from Early Printed Books with Multiple Font Groups}\label{subsubsec:fonts}

This dataset includes over 35,600 images of book pages from the 15th to 18th century. 
These images have been captured in a variety of high resolutions and classified into ten font group classes plus two noise classes~\citep{Seuret19:DatasetEPBFontGroups, Seuret21:ICDARFontGroupDataClassification, Seuret19:ZenodoDatasetoPEPBwMFG}.
The labels are expert-annotated and often multilabel, with strong class imbalance~\citep{Nikolaidou22:SurveyHistoricalDocFontDataset}. The dataset was selected due to its substantial difference from CIFAR-100, both in content and volume, with the former exceeding 44 GB. 
Furthermore, it highlights the importance of energy-efficient network architectures, with direct applications in large-scale historical document classification tasks, such as those conducted in state libraries, where millions of unlabeled pages must be processed. To accommodate GPU memory constraints, we resize training images to $320 \times 320$ for the purpose of achieving better training throughput. 
In addition, the dimensions of the validation and test images are limited to a maximum width of 1000 pixels, thereby restricting validation and test batch sizes to $1$. This dataset is denoted as \textit{Fonts} in the following.

\subsection{Network Architectures}\label{sec:arch:Resnet}

\begin{table}[tb]
    \centering
    \caption{Comparison of our adapted \ac{resnet}-50 architectures to the original.}
    \label{tab:architectures}
    \begin{tabular}{ccc}
    \toprule
         & Original \acs{resnet}-50 & Our \acs{resnet}-50 variants \\
         \midrule
        Convolutional layers & 53 & 65 \\
        Fully-connected layers & 1 & 1 \\
        \textbf{Total layers} & 54 & 66 \\
    \bottomrule
    \end{tabular}
\end{table}

Our models are based on \ac{resnet}-50~\citep{HeZRS15:ResNet}, a residual \ac{cnn} that learns $H(x) - x$ using skip connections.
To alleviate dimensionality dependency between building blocks, we added $1 \times 1$ resampling layers in every building block~\citep{Aghli21:CombiningPruningandKD}. 
Table~\ref{tab:architectures} shows the difference in the number of weight layers from our architectures to the original \ac{resnet}-50. We further use a modified version called \textit{\ac{resnet}50-Bat}, which halves the neurons in the third convolutional layer of each block, and adds dropout before the fully connected layer. The \textit{\ac{resnet}50-Bat} model was employed to attain a higher reference test accuracy than the standard \textit{\ac{resnet}-50} on the \textit{CIFAR-100} dataset. Both models use Leaky\ac{relu} activations~\citep{Khalid20:EmpiricalEvaloActivationFinDCNNsforFacialExprR, Xu15:EmpiricalEvalofReLUinCNNs} to mitigate the dying ReLU problem.
Furthermore, we only apply cropping and rotation as data augmentation.

% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\section{Results}\label{sec:Results}

This section presents the results of our structured pruning method applied to two datasets: \textit{CIFAR-100} and \textit{Fonts}.
We assess both classification accuracy and energy consumption, using sub-networks derived from \textit{\ac{resnet}-50} and \textit{\ac{resnet}50-Bat} models. All networks were trained using the \ac{cel} criterion and the \ac{adam} optimizer~\citep{Kingma17:ADAM}.
Training data was split 80/20 into training and validation sets.
Pruning was applied iteratively with a \textit{neuron pruning percentage} of at least 70\,\% to avoid excessively small pruning steps. No fine-tuning was applied after pruning; hence, reported results reflect raw performance of pruned networks.

For the CIFAR-100 dataset, five pruning runs were performed using \textit{\ac{resnet}50-Bat} on the \textit{CIFAR-100} dataset.
Runs 1 and 2 used 40 training epochs; the remaining runs used 50 to ensure convergence in smaller sub-networks.
Most runs used a 90\,\% \textit{neuron pruning percentage}, with adjustments down to 70\,\% for finer-grained pruning below 83\,\% network size.
Figure~\ref{fig:ResNet50Bat_CIFAR100_NW_SubPlots_range_-0.01} shows that sub-networks around 75\,\% retain comparable test accuracy.
However, pruning beyond this threshold causes performance to drop significantly.
The pruning steps at $\sim$45\,\% network size led to a sharp drop in accuracy, suggesting the removal of critical neurons. The energy consumed per sample reduced by $\sim$50\,\% at around 70\,\% network size.
This is achieved because the absolute energy consumption slightly decreases and the throughput in smaller networks increases. Figure~\ref{fig:ResNet50Bat_CIFAR100_NW_Gains_range_0.6} compares gains in energy and throughput.
Sub-networks around 75\,\% network size reduce energy per sample by $\sim$40\,\% and increase sample count by approximately 50\,\%, while maintaining most of the test accuracy. Figure~\ref{fig:ResNet50Bat_CIFAR100_TestAcc_Energy_range_0.6} summarizes pruning performance: lower energy per sample with only minor losses in test accuracy is preferable.
For example, in \emph{Run 3}, the energy usage decreased from $2.1e^{-5}$ to $0.9e^{-5}$ [Wh/sample] with only 7\, percentage points lost in accuracy.

\begin{figure*}[tb]
    \centering
    \includegraphics[width=0.65\textwidth]{Figures/Add-Exp_ResNet50_Fonts_NW_SubPlots_Wh_range_-0.01.pdf}
    \caption{\ac{resnet}-50 on Fonts: test accuracy, energy usage, and throughput across network sizes.
    The x-axis on all plots shows the network size ranging from 100\,\% to 0\,\%.}
    \label{fig:ResNet50_Fonts_NW_SubPlots_range_-0.01}
\end{figure*}

For the font dataset, three pruning runs were conducted using our adapted \ac{resnet}-50 on the \textit{Fonts} dataset.
Each sub-network was retrained for four epochs between pruning steps.
A 90\,\% \textit{neuron pruning percentage} was used throughout.Figure~\ref{fig:ResNet50_Fonts_NW_SubPlots_range_-0.01} shows that networks pruned below $\sim$40\,\% network size exhibit unstable accuracy, but reduce energy per sample by almost 50\,\%.
Total energy use remains relatively flat due to constant 5-minute GPU runtime at high utilization.Gains shown in Fig.~\ref{fig:ResNet50_Fonts_NW_Gains_range_-0.01} confirm that 40\,\% networks can cut energy per sample by about 40\,\% while increasing throughput by 60\,\% or more and considering only a small decrease in test accuracy.Figure~\ref{fig:ResNet50_Fonts_TestAcc_Energy_range_-0.01} provides the final accuracy/energy trade-off visualization.
Sub-networks with test accuracies above 90\,\% consumed less than half the energy per sample, with \emph{Run 1} delivering particularly strong results.

\begin{figure*}[htbp]
    \centering
    \subfigure[\ac{resnet}-50 on Fonts: Gains in energy per sample and throughput are measured relative to the full network.
    \textit{Gain Energy per Sample} denotes reduced energy use per sample, and \textit{Gain Samples} indicates higher processing throughput.\label{fig:ResNet50_Fonts_NW_Gains_range_-0.01}]{
        \includegraphics[width=0.44\textwidth]{Figures/ResNet50_Fonts_NW_Gains_range_-0.01.pdf}
    }
    \hfill
    \subfigure[\ac{resnet}-50 on Fonts: test accuracy vs. energy per sample.
    The plot shows test accuracy against energy use per sample for all networks. The top box plot marks medians, quantiles, whiskers (1.5 IQR), and outliers.\label{fig:ResNet50_Fonts_TestAcc_Energy_range_-0.01}]{
        \includegraphics[width=0.44\textwidth]{Figures/ResNet50_Fonts_TestAcc_Energy_Wh_range_-0.01.pdf}
    }
    \caption{\ac{resnet}-50 on Fonts}
    % \label{fig:pruning_variants}
\end{figure*}

% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\section{Discussion and Conclusion}

Our structured pruning method achieved over 40\,\% reduction in energy consumption per sample across both architectures and datasets, while maintaining competitive test accuracy.
Some sub-networks even slightly outperformed the full models~--~an effect also reported in~\citet{Frankle19:LTH}~--~though the margins are small. Despite potential modest accuracy drops, they may still be viable for deployment depending on application constraints, particularly when energy efficiency is prioritized. Both \textit{\ac{resnet}-50} and \textit{\ac{resnet}50-Bat} yielded comparable energy savings, with stronger benefits on the \textit{Fonts} dataset.
In this case, \textit{\ac{resnet}-50} was more stable under aggressive pruning.
From Figures~\ref{fig:ResNet50Bat_CIFAR100_NW_SubPlots_range_-0.01} and~\ref{fig:ResNet50_Fonts_NW_SubPlots_range_-0.01}, we note that \textit{\ac{resnet}50-Bat} peaked at roughly $2.3e^{-5}$ [Wh/sample], whereas \textit{\ac{resnet}-50} reached approximately $2.51e^{-4}$ [Wh/sample].
This means that while both models show a 40\,\% gain, the absolute benefit in raw energy usage is higher for \textit{\ac{resnet}-50}.
The improvement amounts to $1.3e^{-5}$ [Wh/sample] for \textit{\ac{resnet}50-Bat} and $1.5e^{-4}$ [Wh / sample] for \textit{\ac{resnet}-50}. One limitation observed is the handling of the \textit{neuron pruning percentage}.
In instances where heavily pruned layers are subsequently selected, minimal alterations in network size occur while potentially crucial neurons are removed.
This can be seen in Fig.~\ref{fig:ResNet50_Fonts_NW_Gains_range_-0.01}, where denser sub-network point clouds sometimes exhibit unexpected drops in accuracy. Overall, these results serve as a baseline for the energy-saving potential of structured pruning methods.
It is important to emphasize that no fine-tuning was applied after pruning, suggesting further improvements are likely achievable.
Future work may incorporate methods like \ac{eb} tickets~\citep{You19:DrawingEBTicketsEfficientTraining} or pruning mechanisms as proposed in~\citet{Rachwan22:WinningLTHAhead}, which dynamically select the \textit{neuron pruning percentage} and may address current limitations. We envision broader adoption of structured pruning in real-world scenarios.
A practical example would be national or academic libraries, where neural networks are applied at scale for classification tasks in digitized historical archives.
Energy-efficient models can play a crucial role in reducing operational costs and environmental impact, especially as the energy sector remains responsible for approximately 34\,\% of global greenhouse gas emissions~\citep{Lee23:IPCC}. For our future research, we plan to explore quantization alongside pruning~\citep{Liang21:PruningQuantizationDNNSurvey}, as well as dynamic pruning strategies where metrics like the \textit{neuron pruning percentage} are adjusted iteratively based on feedback during pruning.


\bibliography{egbib}
\end{document}
