Skip to content

Commit d5aea45

Browse files
ZhaoqiongZjimmytwei
ZhaoqiongZ
and
jimmytwei
authored
add IntelPytorch Quantization code samples (#1301)
* add IntelPytorch Quantization code samples * fix the spelling error in the README file * use john's README with grammar fix and title change * Rename third-party-grograms.txt to third-party-programs.txt Co-authored-by: Jimmy Wei <[email protected]>
1 parent 303ab84 commit d5aea45

File tree

8 files changed

+1221
-0
lines changed

8 files changed

+1221
-0
lines changed

‎AI-and-Analytics/Features-and-Functionality/IntelPytorch_Quantization/IntelPytorch_Quantization.ipynb

+617
Large diffs are not rendered by default.

‎AI-and-Analytics/Features-and-Functionality/IntelPytorch_Quantization/IntelPytorch_Quantization.py

+154
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,154 @@
1+
#!/usr/bin/env python
2+
# encoding: utf-8
3+
4+
'''
5+
==============================================================
6+
Copyright © 2022 Intel Corporation
7+
SPDX-License-Identifier: MIT
8+
==============================================================
9+
'''
10+
11+
import torch
12+
import torchvision
13+
import tqdm
14+
import os
15+
from time import time
16+
import intel_extension_for_pytorch as ipex
17+
from intel_extension_for_pytorch.quantization import prepare, convert
18+
19+
# Hyperparameters and constants
20+
LR = 0.001
21+
DOWNLOAD = True
22+
DATA = 'datasets/cifar10/'
23+
WARMUP = 3
24+
ITERS = 100
25+
26+
27+
def inference(model, data):
28+
# Warmup for several iteration.
29+
for i in range(WARMUP):
30+
out = model(data)
31+
32+
# Benchmark: accumulate inference time for multi iteration and calculate the average inference time.
33+
print("Inference ...")
34+
inference_time = 0
35+
for i in range(ITERS):
36+
start_time = time()
37+
_ = model(data)
38+
end_time = time()
39+
inference_time = inference_time + (end_time - start_time)
40+
41+
42+
43+
inference_time = inference_time / ITERS
44+
print("Inference Time Avg: ", inference_time)
45+
return inference_time
46+
47+
48+
def staticQuantize(model_fp32, data, calibration_data_loader):
49+
# Acquire inference times for static quantization INT8 model
50+
qconfig_static = ipex.quantization.default_static_qconfig
51+
# Alternatively, define your own qconfig:
52+
# from torch.ao.quantization import MinMaxObserver, PerChannelMinMaxObserver, QConfig
53+
# qconfig = QConfig(activation=MinMaxObserver.with_args(qscheme=torch.per_tensor_affine, dtype=torch.quint8),
54+
# weight=PerChannelMinMaxObserver.with_args(dtype=torch.qint8, qscheme=torch.per_channel_symmetric))
55+
prepared_model_static = prepare(model_fp32, qconfig_static, example_inputs=data, inplace=False)
56+
print("Calibration with Static Quantization ...")
57+
for batch_idx, (data, target) in enumerate(calibration_data_loader):
58+
prepared_model_static(data)
59+
if batch_idx % 10 == 0:
60+
print("Batch %d/%d complete, continue ..." %(batch_idx+1, len(calibration_data_loader)))
61+
print("Calibration Done")
62+
63+
converted_model_static = convert(prepared_model_static)
64+
with torch.no_grad():
65+
traced_model_static = torch.jit.trace(converted_model_static, data)
66+
traced_model_static = torch.jit.freeze(traced_model_static)
67+
68+
traced_model_static.save("quantized_model_static.pt")
69+
return traced_model_static
70+
71+
def dynamicQuantize(model_fp32, data):
72+
# Acquire inference times for dynamic quantization INT8 model
73+
qconfig_dynamic = ipex.quantization.default_dynamic_qconfig
74+
print("Quantize Model with Dynamic Quantization ...")
75+
76+
prepared_model_dynamic = prepare(model_fp32, qconfig_dynamic, example_inputs=data, inplace=False)
77+
78+
converted_model_dynamic = convert(prepared_model_dynamic)
79+
with torch.no_grad():
80+
traced_model_dynamic = torch.jit.trace(converted_model_dynamic, data)
81+
traced_model_dynamic = torch.jit.freeze(traced_model_dynamic)
82+
83+
# save the quantized static model
84+
traced_model_dynamic.save("quantized_model_dynamic.pt")
85+
return traced_model_dynamic
86+
87+
88+
"""
89+
Perform all types of training in main function
90+
"""
91+
def main():
92+
93+
# Load dataset
94+
transform = torchvision.transforms.Compose([
95+
torchvision.transforms.Resize((224, 224)),
96+
torchvision.transforms.ToTensor(),
97+
torchvision.transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))
98+
])
99+
test_dataset = torchvision.datasets.CIFAR10(
100+
root=DATA,
101+
train=False,
102+
transform=transform,
103+
download=DOWNLOAD,
104+
)
105+
calibration_data_loader = torch.utils.data.DataLoader(
106+
dataset=test_dataset,
107+
batch_size=128
108+
)
109+
110+
data = torch.rand(1, 3, 224, 224)
111+
model_fp32 = torchvision.models.resnet50(weights=torchvision.models.ResNet50_Weights.DEFAULT)
112+
model_fp32.eval()
113+
114+
if not os.path.exists('quantized_model_static.pt'):
115+
# Static Quantizaton & Save Model to quantized_model_static.pt
116+
print('quantize the model with static quantization')
117+
staticQuantize(model_fp32, data, calibration_data_loader)
118+
119+
if not os.path.exists('quantized_model_dynamic.pt'):
120+
# Dynamic Quantization & Save Model to quantized_model_dynamic.pt
121+
print('quantize the model with dynamic quantization')
122+
dynamicQuantize(model_fp32, data)
123+
124+
print("Inference with FP32")
125+
fp32_inference_time = inference(model_fp32, data)
126+
127+
traced_model_static = torch.jit.load('quantized_model_static.pt')
128+
traced_model_static.eval()
129+
traced_model_static = torch.jit.freeze(traced_model_static)
130+
print("Inference with Static INT8")
131+
int8_inference_time_static = inference(traced_model_static, data)
132+
133+
traced_model_dynamic = torch.jit.load('quantized_model_dynamic.pt')
134+
traced_model_dynamic.eval()
135+
traced_model_dynamic = torch.jit.freeze(traced_model_dynamic)
136+
print("Inference with Dynamic INT8")
137+
int8_inference_time_dynamic = inference(traced_model_dynamic, data)
138+
139+
# Inference time results
140+
print("Summary")
141+
print("FP32 inference time: %.3f" %fp32_inference_time)
142+
print("INT8 static quantization inference time: %.3f" %int8_inference_time_static)
143+
print("INT8 dynamic quantization inference time: %.3f" %int8_inference_time_dynamic)
144+
145+
# Calculate speedup when using quantization
146+
speedup_from_fp32_static = fp32_inference_time / int8_inference_time_static
147+
print("Staic INT8 %.2fX faster than FP32" %speedup_from_fp32_static)
148+
speedup_from_fp32_dynamic = fp32_inference_time / int8_inference_time_dynamic
149+
print("Dynamic INT8 %.2fX faster than FP32" %speedup_from_fp32_dynamic)
150+
151+
152+
if __name__ == '__main__':
153+
main()
154+
print('[CODE_SAMPLE_COMPLETED_SUCCESFULLY]')

‎AI-and-Analytics/Features-and-Functionality/IntelPytorch_Quantization/License.txt

+7
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
Copyright Intel Corporation
2+
3+
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
4+
5+
The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
6+
7+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

‎AI-and-Analytics/Features-and-Functionality/IntelPytorch_Quantization/README.md

+159
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,159 @@
1+
# `Optimize PyTorch Models using Intel® Extension for PyTorch* Quantization` Sample
2+
3+
The `Optimize PyTorch Models using Intel® Extension for PyTorch* Quantization` sample demonstrates how to quantize a ResNet50 model that is calibrated by the CIFAR10 dataset using the Intel® Extension for PyTorch*.
4+
5+
The Intel® Extension for PyTorch* extends PyTorch* with optimizations for extra performance boost on Intel® hardware. While most of the optimizations will be included in future PyTorch* releases, the extension delivers up-to-date features and optimizations for PyTorch on Intel® hardware. For example, newer optimizations include AVX-512 Vector Neural Network Instructions (AVX512 VNNI) and Intel® Advanced Matrix Extensions (Intel® AMX).
6+
7+
| Area | Description
8+
|:--- |:---
9+
| What you will learn | Inference performance improvements using Intel® Extension for PyTorch* (IPEX) with feature quantization
10+
| Time to complete | 5 minutes
11+
| Category | Concepts and Functionality
12+
13+
## Purpose
14+
15+
The Intel® Extension for PyTorch* gives users the ability to speed up inference on Intel® Xeon Scalable processors with INT8 data format and specialized computer instructions. The INT8 data format uses quarter the bit width of floating-point-32 (FP32), lowering the amount of memory needed and execution time to process.
16+
17+
## Prerequisites
18+
19+
| Optimized for | Description
20+
|:--- |:---
21+
| OS | Ubuntu* 18.04 or newer
22+
| Hardware | Intel® Xeon® Scalable Processor family
23+
| Software | Intel® Extension for PyTorch*
24+
25+
### For Local Development Environments
26+
27+
You will need to download and install the following toolkits, tools, and components to use the sample.
28+
29+
- **Intel® AI Analytics Toolkit (AI Kit)**
30+
31+
You can get the AI Kit from [Intel® oneAPI Toolkits](https://www.intel.com/content/www/us/en/developer/tools/oneapi/toolkits.html#analytics-kit). <br> See [*Get Started with the Intel® AI Analytics Toolkit for Linux**](https://www.intel.com/content/www/us/en/develop/documentation/get-started-with-ai-linux) for AI Kit installation information and post-installation steps and scripts.
32+
33+
- **Jupyter Notebook**
34+
35+
Install using PIP: `$pip install notebook`. <br> Alternatively, see [*Installing Jupyter*](https://jupyter.org/install) for detailed installation instructions.
36+
37+
- **Additional Packages**
38+
39+
You will need to install these additional packages: **Matplotlib** and **tqdm**.
40+
```
41+
python -m pip install matplotlib
42+
python -m pip install tqdm
43+
```
44+
45+
### For Intel® DevCloud
46+
47+
The necessary tools and components are already installed in the environment. You do not need to install additional components. See *[Intel® DevCloud for oneAPI](https://DevCloud.intel.com/oneapi/get_started/)* for information.
48+
49+
## Key Implementation Details
50+
51+
This code sample quantizes a ResNet50 model that is calibrated with the CIFAR10 dataset while using Intel® Extension for PyTorch*. The model is inferenced using FP32 and INT8 precision, including the use of Intel® Advanced Matrix Extensions (AMX). AMX is supported on BF16 and INT8 data types starting with 4th Gen Xeon Scalable Processors. The inference time will be compared, showcasing the speedup of INT8.
52+
53+
The sample tutorial contains one Jupyter Notebook and a Python script. You can use either.
54+
55+
### Jupyter Notebook
56+
57+
| Notebook | Description
58+
|:--- |:---
59+
|`IntelPyTorch_Quantization.ipynb` | Performs inference with IPEX quantization in Jupyter Notebook.
60+
61+
### Python Scripts
62+
63+
| Script | Description
64+
|:--- |:---
65+
|`IntelPyTorch_Quantization.py` | The script performs inference with IPEX quantization and compares the performance against the baseline
66+
67+
## Set Environment Variables
68+
69+
When working with the command-line interface (CLI), you should configure the oneAPI toolkits using environment variables. Set up your CLI environment by sourcing the `setvars` script every time you open a new terminal window. This practice ensures that your compiler, libraries, and tools are ready for development.
70+
71+
## Run the `Optimize PyTorch Models using Intel® Extension for PyTorch* Quantization` Sample
72+
73+
### On Linux*
74+
75+
> **Note**: If you have not already done so, set up your CLI
76+
> environment by sourcing the `setvars` script in the root of your oneAPI installation.
77+
>
78+
> Linux*:
79+
> - For system wide installations: `. /opt/intel/oneapi/setvars.sh`
80+
> - For private installations: ` . ~/intel/oneapi/setvars.sh`
81+
> - For non-POSIX shells, like csh, use the following command: `bash -c 'source <install-dir>/setvars.sh ; exec csh'`
82+
>
83+
> For more information on configuring environment variables, see *[Use the setvars Script with Linux* or macOS*](https://www.intel.com/content/www/us/en/develop/documentation/oneapi-programming-guide/top/oneapi-development-environment-setup/use-the-setvars-script-with-linux-or-macos.html)*.
84+
#### Activate Conda
85+
86+
1. Activate the Conda environment.
87+
```
88+
conda activate pytorch
89+
```
90+
2. Activate Conda environment without Root access (Optional).
91+
92+
By default, the AI Kit is installed in the `/opt/intel/oneapi` folder and requires root privileges to manage it.
93+
94+
You can choose to activate Conda environment without root access. To bypass root access to manage your Conda environment, clone and activate your desired Conda environment using the following commands similar to the following.
95+
96+
```
97+
conda create --name user_pytorch --clone pytorch
98+
conda activate user_pytorch
99+
```
100+
101+
#### Running the Jupyter Notebook
102+
103+
1. Change to the sample directory.
104+
2. Launch Jupyter Notebook.
105+
```
106+
jupyter notebook --ip=0.0.0.0 --port 8888 --allow-root
107+
```
108+
3. Follow the instructions to open the URL with the token in your browser.
109+
4. Locate and select the Notebook.
110+
```
111+
IntelPyTorch_Quantization.ipynb
112+
```
113+
5. Change your Jupyter Notebook kernel to **PyTorch (AI kit)**.
114+
6. Run every cell in the Notebook in sequence.
115+
116+
#### Running on the Command Line (Optional)
117+
118+
1. Change to the sample directory.
119+
2. Run the script.
120+
```
121+
python IntelPyTorch_Quantization.py
122+
```
123+
124+
### Run the `Optimize PyTorch Models using Intel® Extension for PyTorch* Quantization` Sample on Intel® DevCloud
125+
126+
1. If you do not already have an account, request an Intel® DevCloud account at [*Create an Intel® DevCloud Account*](https://intelsoftwaresites.secure.force.com/DevCloud/oneapi).
127+
2. On a Linux* system, open a terminal.
128+
3. SSH into Intel® DevCloud.
129+
```
130+
ssh DevCloud
131+
```
132+
> **Note**: You can find information about configuring your Linux system and connecting to Intel DevCloud at Intel® DevCloud for oneAPI [Get Started](https://DevCloud.intel.com/oneapi/get_started).
133+
134+
4. Follow the instructions to open the URL with the token in your browser.
135+
5. Locate and select the Notebook.
136+
```
137+
IntelPyTorch_Quantization.ipynb
138+
````
139+
6. Change the kernel to **PyTorch (AI kit)**.
140+
7. Run every cell in the Notebook in sequence.
141+
142+
### Troubleshooting
143+
144+
If you receive an error message, troubleshoot the problem using the **Diagnostics Utility for Intel® oneAPI Toolkits**. The diagnostic utility provides configuration and system checks to help find missing dependencies, permissions errors, and other issues. See the *[Diagnostics Utility for Intel® oneAPI Toolkits User Guide](https://www.intel.com/content/www/us/en/develop/documentation/diagnostic-utility-user-guide/top.html)* for more information on using the utility.
145+
146+
## Example Output
147+
148+
If successful, the sample displays `[CODE_SAMPLE_COMPLETED_SUCCESSFULLY]`. Additionally, the sample generates performance and analysis diagrams for comparison.
149+
150+
The following image shows approximate performance speed increases using AMX INT8 during inference.
151+
152+
![](assets/relative_speedup.png)
153+
154+
## License
155+
156+
Code samples are licensed under the MIT license. See
157+
[License.txt](https://github.com/oneapi-src/oneAPI-samples/blob/master/License.txt) for details.
158+
159+
Third party program Licenses can be found here: [third-party-programs.txt](https://github.com/oneapi-src/oneAPI-samples/blob/master/third-party-programs.txt)

‎AI-and-Analytics/Features-and-Functionality/IntelPytorch_Quantization/assets/relative_speedup.png

28.8 KB
Loading

‎AI-and-Analytics/Features-and-Functionality/IntelPytorch_Quantization/requirements.txt

+2
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
tqdm
2+
matplotlib

‎AI-and-Analytics/Features-and-Functionality/IntelPytorch_Quantization/sample.json

+29
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
{
2+
"guid": "54B3F469-1E58-400B-9FCA-8BE08F680DCA",
3+
"name": "Optimize PyTorch Models using Intel® Extension for PyTorch* (IPEX) Quantization",
4+
"categories": ["Toolkit/oneAPI AI And Analytics/Features And Functionality"],
5+
"description": "Applying IPEX Quantization Optimizations to a PyTorch workload in a step-by-step manner to gain performance boost in inference.",
6+
"builder": ["cli"],
7+
"languages": [{
8+
"python": {}
9+
}],
10+
"os": ["linux"],
11+
"targetDevice": ["CPU"],
12+
"ciTests": {
13+
"linux": [{
14+
"env": ["source ${ONEAPI_ROOT}/setvars.sh --force",
15+
"conda env remove -n user_pytorch",
16+
"conda create --name user_pytorch --clone pytorch",
17+
"conda activate user_pytorch",
18+
"pip install -r requirements.txt",
19+
"~/.conda/envs/user_pytorch/bin/python -m ipykernel install --user --name=user_pytorch"
20+
],
21+
"id": "ipex_inference_optimization",
22+
"steps": [
23+
"source activate user_pytorch",
24+
"python IntelPytorch_Quantization.py"
25+
]
26+
}]
27+
},
28+
"expertise": "Concepts and Functionality"
29+
}

0 commit comments

Comments
 (0)