Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

Fix example issue (#10)

WeiweiZhang14c1e75d2a0a5
diff --git a/docs/README.md b/docs/README.md
index fd8df5954..9be568441 100644
--- a/docs/README.md
+++ b/docs/README.md
@@ -1,5 +1,2526 @@
For a fair comparison, we utilized 512 samples from Pile-10k for all methods during calibration. Due to memory constraints, we maintained the original sequence length of 512 for AWQ, while for GPTQ and our approach, a sequence length of 2048 is used. We have enalbed act-order and true-seqential in GPTQ, and the notation GPTQ* indicates that we adjusted the random seed or data preprocessing to address issues related to the non-positive definite Hessian matrix or other issues.
-![](W4G-1.png)
-![](W4G128.png)
-![](W3G128.png)
-![](W2G128.png)
+
+</br>
+
+### 1. Accuracies $\uparrow$ across 11 tasks(0-shot) of LLaMA and Mistral models at W4G-1.
+<table border="1">
+ <tr>
+ <td></td>
+ <td></td>
+ <td>Mmlu</td>
+ <td>Lamb.</td>
+ <td>Hella.</td>
+ <td>Wino.</td>
+ <td>Piqa</td>
+ <td>Truth.</td>
+ <td>Open.</td>
+ <td>Boolq</td>
+ <td>RTE</td>
+ <td>ARC-e</td>
+ <td>ARC-c.</td>
+ <td>Avg.</td>
+ </tr>
+ <tr>
+ <td rowspan="5">Mistral-7B</td>
+ <td>FP16</td>
+ <td>61.35</td>
+ <td>75.68</td>
+ <td>61.27</td>
+ <td>74.03</td>
+ <td>80.79</td>
+ <td>28.03</td>
+ <td>32.80</td>
+ <td>83.67</td>
+ <td>67.51</td>
+ <td>80.81</td>
+ <td>50.34</td>
+ <td>63.30</td>
+ </tr>
+ <tr>
+ <td>RTN</td>
+ <td>55.92</td>
+ <td>66.10</td>
+ <td>59.01</td>
+ <td>71.35</td>
+ <td>80.14</td>
+ <td>24.85</td>
+ <td>29.00</td>
+ <td>79.17</td>
+ <td>57.76</td>
+ <td>77.95</td>
+ <td>45.99</td>
+ <td>58.84</td>
+ </tr>
+ <tr>
+ <td>GPTQ</td>
+ <td>58.22</td>
+ <td>73.45</td>
+ <td>59.47</td>
+ <td><b>74.03</td>
+ <td><b>80.20</td>
+ <td>26.93</td>
+ <td>31.00</td>
+ <td>81.50</td>
+ <td>64.98</td>
+ <td>78.24</td>
+ <td>47.01</td>
+ <td>61.37</td>
+ </tr>
+ <tr>
+ <td>AWQ</td>
+ <td>57.20</td>
+ <td>71.45</td>
+ <td>59.21</td>
+ <td>73.64</td>
+ <td>79.43</td>
+ <td>25.34</td>
+ <td>30.40</td>
+ <td><b>82.69</td>
+ <td><b>68.95</td>
+ <td>79.25</td>
+ <td>47.44</td>
+ <td>61.36</td>
+ </tr>
+ <tr>
+ <td>Ours</td>
+ <th>59.52</td>
+ <td><b>73.76</td>
+ <td><b>60.75</td>
+ <td>73.32</td>
+ <td>80.09</td>
+ <td><b>27.17</td>
+ <td><b>33.00</td>
+ <td>82.02</td>
+ <td>66.07</td>
+ <td><b>80.47</td>
+ <td><b>49.49</td>
+ <td><b>62.33</td>
+ </tr>
+ <tr>
+ <td rowspan="5">V2-7B</td>
+ <td>FP16</td>
+ <td>42.69</td>
+ <td>73.90</td>
+ <td>57.15</td>
+ <td>68.90</td>
+ <td>78.07</td>
+ <td>25.21</td>
+ <td>31.40</td>
+ <td>77.74</td>
+ <td>62.82</td>
+ <td>76.35</td>
+ <td>43.52</td>
+ <td>57.98</td>
+ </tr>
+ <tr>
+ <td>RTN</td>
+ <td>36.87</td>
+ <td>67.96</td>
+ <td>55.63</td>
+ <td>68.51</td>
+ <td>76.82</td>
+ <td><b>26.19</td>
+ <td>30.60</td>
+ <td>73.64</td>
+ <td>58.84</td>
+ <td>74.07</td>
+ <td>41.30</td>
+ <td>55.49</td>
+ </tr>
+ <tr>
+ <td>GPTQ</td>
+ <td>39.66</td>
+ <td><b>71.92</td>
+ <td>55.89</td>
+ <td>68.03</td>
+ <td>77.58</td>
+ <td>25.09</td>
+ <td>30.20</td>
+ <td>76.67</td>
+ <td>62.09</td>
+ <td>75.55</td>
+ <td>41.72</td>
+ <td>56.76</td>
+ </tr>
+ <tr>
+ <td>AWQ</td>
+ <td><b>40.24</td>
+ <td>71.20</td>
+ <td>56.26</td>
+ <td><b>69.61</td>
+ <td>76.93</td>
+ <td>26.07</td>
+ <td><b>32.60</td>
+ <td><b>77.31</td>
+ <td>63.18</td>
+ <td>75.00</td>
+ <td>41.30</td>
+ <td>57.25</td>
+ </tr>
+ <tr>
+ <td>Ours</td>
+ <td>39.97</td>
+ <td>71.63</td>
+ <td><b>56.52</td>
+ <td>68.43</td>
+ <td><b>77.91</td>
+ <td>25.70</td>
+ <td>31.60</td>
+ <td>76.18</td>
+ <td><b>65.70</td>
+ <td><b>76.01</td>
+ <td><b>42.58</td>
+ <td><b>57.48</td>
+ </tr>
+ <tr>
+ <td rowspan="5">V2-13B</td>
+ <td>FP16</td>
+ <td>52.86</td>
+ <td>76.77</td>
+ <td>60.04</td>
+ <td>72.14</td>
+ <td>79.05</td>
+ <td>25.95</td>
+ <td>35.20</td>
+ <td>80.55</td>
+ <td>65.34</td>
+ <td>79.38</td>
+ <td>48.38</td>
+ <td>61.42</td>
+ </tr>
+ <tr>
+ <td>RTN</td>
+ <td>50.37</td>
+ <td>74.35</td>
+ <td>59.12</td>
+ <td>71.98</td>
+ <td><b>79.00</td>
+ <td>24.85</td>
+ <td>33.00</td>
+ <td><b>81.77</td>
+ <td>64.98</td>
+ <td>79.08</td>
+ <td>46.59</td>
+ <td>60.46</td>
+ </tr>
+ <tr>
+ <td>GPTQ</td>
+ <td>51.14</td>
+ <td>75.37</td>
+ <td>59.14</td>
+ <td>72.06</td>
+ <td>78.02</td>
+ <td>25.34</td>
+ <td>32.20</td>
+ <td>80.46</td>
+ <td>62.09</td>
+ <td>77.36</td>
+ <td>44.54</td>
+ <td>59.79</td>
+ </tr>
+ <tr>
+ <td>AWQ</td>
+ <td>51.16</td>
+ <td><b>75.98</td>
+ <td>59.51</td>
+ <td>70.80</td>
+ <td>78.40</td>
+ <td>25.21</td>
+ <td><b>34.60</td>
+ <td>78.26</td>
+ <td><b>66.79</td>
+ <td>79.12</td>
+ <td>46.59</td>
+ <td>60.58</td>
+ </tr>
+ <tr>
+ <td>Ours</td>
+ <td><b>52.30</td>
+ <td>75.96</td>
+ <td><b>59.79</td>
+ <td><b>72.30</td>
+ <td>78.84</td>
+ <td><b>25.58</td>
+ <td>34.00</td>
+ <td>80.15</td>
+ <td><b>66.79</td>
+ <td><b>79.38</td>
+ <td><b>48.12</td>
+ <td><b>61.20</td>
+ </tr>
+ <tr>
+ <td rowspan="5">V2-70B</td>
+ <td>FP16</td>
+ <td>66.23</td>
+ <td>79.64</td>
+ <td>64.77</td>
+ <td>77.98</td>
+ <td>82.15</td>
+ <td>30.60</td>
+ <td>37.20</td>
+ <td>83.70</td>
+ <td>67.87</td>
+ <td>82.70</td>
+ <td>54.44</td>
+ <td>66.12</td>
+ </tr>
+ <tr>
+ <td>RTN</td>
+ <td>63.85</td>
+ <td>77.62</td>
+ <td>63.38</td>
+ <td>76.72</td>
+ <td>81.50</td>
+ <td>28.89</td>
+ <td><b>37.80</td>
+ <td>83.39</td>
+ <td>68.23</td>
+ <td>81.99</td>
+ <td>54.10</td>
+ <td>65.22</td>
+ </tr>
+ <tr>
+ <td>GPTQ</td>
+ <td>64.81</td>
+ <td>79.27</td>
+ <td>63.86</td>
+ <td>76.87</td>
+ <td>81.61</td>
+ <td><b>31.46</td>
+ <td>36.40</td>
+ <td>82.23</td>
+ <td>70.04</td>
+ <td><b>82.53</td>
+ <td>54.18</td>
+ <td>65.75</td>
+ </tr>
+ <tr>
+ <td>AWQ</td>
+ <td>65.08</td>
+ <td>78.77</td>
+ <td>64.14</td>
+ <td>77.11</td>
+ <td>81.45</td>
+ <td>30.48</td>
+ <td>37.20</td>
+ <td>83.64</td>
+ <td><b>72.92</td>
+ <td>82.49</td>
+ <td><b>55.80</td>
+ <td><b>66.28</td>
+ </tr>
+ <tr>
+ <td>Ours</td>
+ <td><b>65.43</td>
+ <td><b>79.55</td>
+ <td><b>64.47</td>
+ <td><b>78.06</td>
+ <td><b>82.10</td>
+ <td>30.60</td>
+ <td>36.40</td>
+ <td><b>83.91</td>
+ <td>71.12</td>
+ <td><b>82.53</td>
+ <td>54.78</td>
+ <td>66.27</td>
+ </tr>
+ <tr>
+ <td rowspan="5">V1-7B</td>
+ <td>FP16</td>
+ <td>32.74</td>
+ <td>73.53</td>
+ <td>56.94</td>
+ <td>70.01</td>
+ <td>78.67</td>
+ <td>22.03</td>
+ <td>34.60</td>
+ <td>75.08</td>
+ <td>66.43</td>
+ <td>75.25</td>
+ <td>41.81</td>
+ <td>57.01</td>
+ </tr>
+ <tr>
+ <td>RTN</td>
+ <td>31.34</td>
+ <td>70.02</td>
+ <td>55.35</td>
+ <td>69.77</td>
+ <td>77.69</td>
+ <td>20.32</td>
+ <td>32.60</td>
+ <td>73.43</td>
+ <td>59.57</td>
+ <td>74.45</td>
+ <td>41.30</td>
+ <td>55.08</td>
+ </tr>
+ <tr>
+ <td>GPTQ</td>
+ <td>29.06</td>
+ <td>71.08</td>
+ <td>55.11</td>
+ <td><b>70.01</td>
+ <td>77.37</td>
+ <td>20.93</td>
+ <td>32.20</td>
+ <td>72.69</td>
+ <td>63.90</td>
+ <td>74.66</td>
+ <td>41.64</td>
+ <td>55.33</td>
+ </tr>
+ <tr>
+ <td>AWQ</td>
+ <td><b>33.33</td>
+ <td>70.81</td>
+ <td>55.98</td>
+ <td>68.27</td>
+ <td>78.07</td>
+ <td>21.18</td>
+ <td>31.40</td>
+ <td>74.37</td>
+ <td>64.62</td>
+ <td>74.03</td>
+ <td>41.21</td>
+ <td>55.75</td>
+ </tr>
+ <tr>
+ <td>Ours</td>
+ <td>31.80</td>
+ <td><b>71.96</td>
+ <td><b>56.57</td>
+ <td>69.53</td>
+ <td><b>79.00</td>
+ <td><b>21.91</td>
+ <td><b>33.20</td>
+ <td><b>75.72</td>
+ <td><b>66.79</td>
+ <td><b>74.83</td>
+ <td><b>43.09</td>
+ <td><b>56.76</td>
+ </tr>
+ <tr>
+ <td rowspan="5">V1-13B</td>
+ <td>FP16</td>
+ <td>44.21</td>
+ <td>76.21</td>
+ <td>59.92</td>
+ <td>72.77</td>
+ <td>79.16</td>
+ <td>25.70</td>
+ <td>33.20</td>
+ <td>77.89</td>
+ <td>70.76</td>
+ <td>77.40</td>
+ <td>46.42</td>
+ <td>60.33</td>
+ </tr>
+ <tr>
+ <td>RTN</td>
+ <td>39.57</td>
+ <td>70.93</td>
+ <td>58.82</td>
+ <td>71.98</td>
+ <td>78.02</td>
+ <td>24.85</td>
+ <td>32.00</td>
+ <td><b>78.20</td>
+ <td>66.43</td>
+ <td>75.67</td>
+ <td>44.62</td>
+ <td>58.28</td>
+ </tr>
+ <tr>
+ <td>GPTQ*</td>
+ <td>40.01</td>
+ <td>74.67</td>
+ <td>58.92</td>
+ <td>71.03</td>
+ <td>78.45</td>
+ <td><b>26.44</td>
+ <td><b>33.60</td>
+ <td>77.09</td>
+ <td><b>68.23</td>
+ <td>76.85</td>
+ <td>44.97</td>
+ <td>59.12</td>
+ </tr>
+ <tr>
+ <td>AWQ</td>
+ <td><b>44.56</td>
+ <td>74.13</td>
+ <td>59.13</td>
+ <td>71.27</td>
+ <td><b>78.94</td>
+ <td>25.83</td>
+ <td>33.20</td>
+ <td>76.42</td>
+ <td>66.06</td>
+ <td><b>76.89</td>
+ <td><b>46.67</td>
+ <td>59.37</td>
+ </tr>
+ <tr>
+ <td>Ours</td>
+ <td>43.94</td>
+ <td><b>75.82</td>
+ <td><b>59.51</td>
+ <td><b>72.22</td>
+ <td>78.78</td>
+ <td>25.70</td>
+ <td>32.80</td>
+ <td>77.34</td>
+ <td>67.51</td>
+ <td>76.47</td>
+ <td><b>46.67</td>
+ <td><b>59.71</td>
+ </tr>
+ <tr>
+ <td rowspan="5">V1-30B</td>
+ <td>FP16</td>
+ <td>55.14</td>
+ <td>77.55</td>
+ <td>63.33</td>
+ <td>75.85</td>
+ <td>81.12</td>
+ <td>28.27</td>
+ <td>36.00</td>
+ <td>82.78</td>
+ <td>66.79</td>
+ <td>80.39</td>
+ <td>52.90</td>
+ <td>63.65</td>
+ </tr>
+ <tr>
+ <td>RTN</td>
+ <td>53.05</td>
+ <td>75.65</td>
+ <td>62.08</td>
+ <td>74.82</td>
+ <td>80.09</td>
+ <td>25.95</td>
+ <td>35.80</td>
+ <td>81.87</td>
+ <td>63.54</td>
+ <td>79.76</td>
+ <td>50.26</td>
+ <td>62.08</td>
+ </tr>
+ <tr>
+ <td>GPTQ</td>
+ <td>53.04</td>
+ <td>77.22</td>
+ <td>61.95</td>
+ <td>73.80</td>
+ <td>80.69</td>
+ <td>27.29</td>
+ <td>34.60</td>
+ <td>81.07</td>
+ <td>66.06</td>
+ <td>78.79</td>
+ <td>49.15</td>
+ <td>62.15</td>
+ </tr>
+ <tr>
+ <td>AWQ</td>
+ <td>54.13</td>
+ <td>76.77</td>
+ <td>62.78</td>
+ <td>74.11</td>
+ <td><b>81.07</td>
+ <td><b>27.78</td>
+ <td>35.00</td>
+ <td><b>82.66</td>
+ <td><b>67.15</td>
+ <td>79.97</td>
+ <td>51.71</td>
+ <td>63.01</td>
+ </tr>
+ <tr>
+ <td>Ours</td>
+ <td><b>54.72</td>
+ <td><b>77.84</td>
+ <td><b>62.91</td>
+ <td><b>75.06</td>
+ <td>80.69</td>
+ <td>26.68</td>
+ <td><b>36.40</td>
+ <td>82.60</td>
+ <td>66.79</td>
+ <td><b>80.13</td>
+ <td><b>52.13</td>
+ <td><b>63.27</td>
+ </tr>
+ <tr>
+ <td rowspan="5">V1-65B</td>
+ <td>FP16</td>
+ <td>59.79</td>
+ <td>79.12</td>
+ <td>64.53</td>
+ <td>77.35</td>
+ <td>81.23</td>
+ <td>27.91</td>
+ <td>38.00</td>
+ <td>84.86</td>
+ <td>69.68</td>
+ <td>81.36</td>
+ <td>52.82</td>
+ <td>65.15</td>
+ </tr>
+ <tr>
+ <td>RTN</td>
+ <td>58.74</td>
+ <td>76.42</td>
+ <td>64.12</td>
+ <td><b>76.72</td>
+ <td>81.01</td>
+ <td><b>29.25</td>
+ <td><b>38.60</td>
+ <td>84.13</td>
+ <td>70.40</td>
+ <td>80.72</td>
+ <td><b>51.88</td>
+ <td>64.73</td>
+ </tr>
+ <tr>
+ <td>GPTQ*</td>
+ <td>59.10</td>
+ <td>78.17</td>
+ <td>63.78</td>
+ <td>75.69</td>
+ <td><b>81.34</td>
+ <td>28.27</td>
+ <td>38.40</td>
+ <td>83.76</td>
+ <td>68.59</td>
+ <td><b>80.98</td>
+ <td>51.62</td>
+ <td>64.52</td>
+ </tr>
+ <tr>
+ <td>AWQ</td>
+ <td>58.86</td>
+ <td>77.37</td>
+ <td>63.86</td>
+ <td>76.56</td>
+ <td>80.85</td>
+ <td>28.27</td>
+ <td>35.20</td>
+ <td>83.94</td>
+ <td><b>71.48</td>
+ <td>78.75</td>
+ <td>50.94</td>
+ <td>64.19</td>
+ </tr>
+ <tr>
+ <td>Ours</td>
+ <td><b>59.21</td>
+ <td><b>79.16</td>
+ <td><b>64.37</td>
+ <td>76.64</td>
+ <td><b>81.34</td>
+ <td>26.81</td>
+ <td>37.80</td>
+ <td><b>84.40</td>
+ <td>69.68</td>
+ <td><b>80.98</td>
+ <td>51.79</td>
+ <td><b>64.74</td>
+ </tr>
+</table>
+
+</br>
+
+### 2. Accuracies $\uparrow$ across 11 tasks(0-shot) of LLaMA and Mistral models at W4G128.
+
+<table border="1">
+ <tr>
+ <td></td>
+ <td></td>
+ <td>Mmlu</td>
+ <td>Lamb.</td>
+ <td>Hella.</td>
+ <td>Wino.</td>
+ <td>Piqa</td>
+ <td>Truth.</td>
+ <td>Open.</td>
+ <td>Boolq</td>
+ <td>RTE</td>
+ <td>ARC-e</td>
+ <td>ARC-c.</td>
+ <td>Avg.</td>
+ </tr>
+ <tr>
+ <td rowspan="5">Mistral-7B</td>
+ <td>FP16</td>
+ <td>61.35</td>
+ <td>75.68</td>
+ <td>61.27</td>
+ <td>74.03</td>
+ <td>80.79</td>
+ <td>28.03</td>
+ <td>32.80</td>
+ <td>83.67</td>
+ <td>67.51</td>
+ <td>80.81</td>
+ <td>50.34</td>
+ <td>63.30</td>
+ </tr>
+ <tr>
+ <td>RTN</td>
+ <td>59.72</td>
+ <td>74.44</td>
+ <td><b>61.06</td>
+ <td>73.40</td>
+ <td>80.36</td>
+ <td>27.17</td>
+ <td><b>32.60</td>
+ <td>83.67</td>
+ <td>64.62</td>
+ <td>79.63</td>
+ <td>49.32</td>
+ <td>62.36</td>
+ </tr>
+ <tr>
+ <td>GPTQ</td>
+ <td>59.17</td>
+ <td>74.52</td>
+ <td>60.37</td>
+ <td><b>74.90</td>
+ <td><b>80.58</td>
+ <td>26.68</td>
+ <td>31.00</td>
+ <td>83.33</td>
+ <td><b>67.15</td>
+ <td>79.67</td>
+ <td>48.12</td>
+ <td>62.32</td>
+ </tr>
+ <tr>
+ <td>AWQ</td>
+ <td>60.20</td>
+ <td>75.14</td>
+ <td>60.43</td>
+ <td>73.80</td>
+ <td>80.03</td>
+ <td>27.05</td>
+ <td>30.40</td>
+ <td><b>84.01</td>
+ <td>62.09</td>
+ <td><b>80.39</td>
+ <td><b>50.26</td>
+ <td>62.16</td>
+ </tr>
+ <tr>
+ <td>Ours</td>
+ <td><b>60.47</td>
+ <td><b>75.59</td>
+ <td>61.03</td>
+ <td>73.88</td>
+ <td>80.09</td>
+ <td><b>27.54</td>
+ <td>31.60</td>
+ <td>83.09</td>
+ <td>66.07</td>
+ <td>79.97</td>
+ <td>49.49</td>
+ <td><b>62.62</td>
+ </tr>
+ <tr>
+ <td rowspan="5">V2-7B</td>
+ <td>FP16</td>
+ <td>42.69</td>
+ <td>73.90</td>
+ <td>57.15</td>
+ <td>68.90</td>
+ <td>78.07</td>
+ <td>25.21</td>
+ <td>31.40</td>
+ <td>77.74</td>
+ <td>62.82</td>
+ <td>76.35</td>
+ <td>43.52</td>
+ <td>57.98</td>
+ </tr>
+ <tr>
+ <td>RTN</td>
+ <td>40.91</td>
+ <td>72.44</td>
+ <td><b>56.91</td>
+ <td>68.35</td>
+ <td>77.58</td>
+ <td>24.97</td>
+ <td>31.20</td>
+ <td>77.61</td>
+ <td>56.32</td>
+ <td><b>76.26</td>
+ <td>43.52</td>
+ <td>56.92</td>
+ </tr>
+ <tr>
+ <td>GPTQ</td>
+ <td><b>42.57</td>
+ <td><b>73.28</td>
+ <td>56.36</td>
+ <td><b>69.06</td>
+ <td>78.02</td>
+ <td>25.34</td>
+ <td>30.20</td>
+ <td>75.72</td>
+ <td>57.04</td>
+ <td>75.63</td>
+ <td>42.15</td>
+ <td>56.85</td>
+ </tr>
+ <tr>
+ <td>AWQ</td>
+ <td>41.00</td>
+ <td>72.60</td>
+ <td>56.40</td>
+ <td>68.98</td>
+ <td>77.31</td>
+ <td><b>25.70</td>
+ <td><b>31.60</td>
+ <td><b>78.75</td>
+ <td>58.48</td>
+ <td>76.14</td>
+ <td><b>43.86</td>
+ <td>57.35</td>
+ </tr>
+ <tr>
+ <td>Ours</td>
+ <td>41.82</td>
+ <td>72.75</td>
+ <td>56.79</td>
+ <td>68.67</td>
+ <td><b>78.13</td>
+ <td>25.58</td>
+ <td>30.20</td>
+ <td>77.49</td>
+ <td><b>63.54</td>
+ <td>75.76</td>
+ <td>42.58</td>
+ <td><b>57.57</td>
+ </tr>
+ <tr>
+ <td rowspan="5">V2-13B</td>
+ <td>FP16</td>
+ <td>52.86</td>
+ <td>76.77</td>
+ <td>60.04</td>
+ <td>72.14</td>
+ <td>79.05</td>
+ <td>25.95</td>
+ <td>35.20</td>
+ <td>80.55</td>
+ <td>65.34</td>
+ <td>79.38</td>
+ <td>48.38</td>
+ <td>61.42</td>
+ </tr>
+ <tr>
+ <td>RTN</td>
+ <td>52.10</td>
+ <td>76.27</td>
+ <td>59.77</td>
+ <td>72.14</td>
+ <td>78.62</td>
+ <td>24.72</td>
+ <td>34.20</td>
+ <td>80.24</td>
+ <td>62.09</td>
+ <td>79.00</td>
+ <td><b>47.95</td>
+ <td>60.65</td>
+ </tr>
+ <tr>
+ <td>GPTQ</td>
+ <td><b>52.66</td>
+ <td>76.54</td>
+ <td>59.76</td>
+ <td>72.14</td>
+ <td>78.35</td>
+ <td>25.70</td>
+ <td>34.00</td>
+ <td>79.33</td>
+ <td><b>66.43</td>
+ <td>78.58</td>
+ <td>47.53</td>
+ <td><b>61.00</td>
+ </tr>
+ <tr>
+ <td>AWQ</td>
+ <td>52.39</td>
+ <td><b>76.89</td>
+ <td><b>59.97</td>
+ <td><b>73.24</td>
+ <td><b>79.00</td>
+ <td>25.21</td>
+ <td>32.60</td>
+ <td><b>80.40</td>
+ <td>63.54</td>
+ <td>79.04</td>
+ <td>47.70</td>
+ <td>60.91</td>
+ </tr>
+ <tr>
+ <td>Ours</td>
+ <td>51.92</td>
+ <td>76.46</td>
+ <td>59.87</td>
+ <td>71.67</td>
+ <td><b>79.00</td>
+ <td><b>25.83</td>
+ <td><b>35.20</td>
+ <td>79.60</td>
+ <td>63.54</td>
+ <td><b>79.25</td>
+ <td>47.01</td>
+ <td>60.85</td>
+ </tr>
+ <tr>
+ <td rowspan="5">V2-70B</td>
+ <td>FP16</td>
+ <td>66.23</td>
+ <td>79.64</td>
+ <td>64.77</td>
+ <td>77.98</td>
+ <td>82.15</td>
+ <td>30.60</td>
+ <td>37.20</td>
+ <td>83.70</td>
+ <td>67.87</td>
+ <td>82.70</td>
+ <td>54.44</td>
+ <td>66.12</td>
+ </tr>
+ <tr>
+ <td>RTN</td>
+ <td>64.91</td>
+ <td>79.06</td>
+ <td>63.93</td>
+ <td>78.14</td>
+ <td>81.66</td>
+ <td>30.11</td>
+ <td>37.00</td>
+ <td>83.61</td>
+ <td>68.59</td>
+ <td>82.79</td>
+ <td>54.78</td>
+ <td>65.87</td>
+ </tr>
+ <tr>
+ <td>GPTQ</td>
+ <td>65.63</td>
+ <td>79.22</td>
+ <td>64.45</td>
+ <td>78.22</td>
+ <td>81.88</td>
+ <td>31.09</td>
+ <td>37.00</td>
+ <td>84.19</td>
+ <td><b>69.31</td>
+ <td>82.79</td>
+ <td>54.61</td>
+ <td>66.22</td>
+ </tr>
+ <tr>
+ <td>AWQ</td>
+ <td><b>65.79</td>
+ <td><b>79.76</td>
+ <td>64.48</td>
+ <td>77.58</td>
+ <td><b>82.32</td>
+ <td>30.72</td>
+ <td><b>38.00</td>
+ <td>83.06</td>
+ <td>68.95</td>
+ <td>82.70</td>
+ <td><b>55.12</td>
+ <td>66.23</td>
+ </tr>
+ <tr>
+ <td>Ours</td>
+ <td>65.65</td>
+ <td>79.49</td>
+ <td><b>64.60</td>
+ <td><b>78.30</td>
+ <td>82.05</td>
+ <td><b>31.58</td>
+ <td>37.40</td>
+ <td><b>84.83</td>
+ <td>68.95</td>
+ <td><b>82.87</td>
+ <td>54.52</td>
+ <td><b>66.39</td>
+ </tr>
+ <tr>
+ <td rowspan="5">V1-7B</td>
+ <td>FP16</td>
+ <td>32.74</td>
+ <td>73.53</td>
+ <td>56.94</td>
+ <td>70.01</td>
+ <td>78.67</td>
+ <td>22.03</td>
+ <td>34.60</td>
+ <td>75.08</td>
+ <td>66.43</td>
+ <td>75.25</td>
+ <td>41.81</td>
+ <td>57.01</td>
+ </tr>
+ <tr>
+ <td>RTN</td>
+ <td>32.63</td>
+ <td>72.31</td>
+ <td>56.26</td>
+ <td>70.01</td>
+ <td>78.45</td>
+ <td>20.93</td>
+ <td><b>33.60</td>
+ <td>74.74</td>
+ <td>64.26</td>
+ <td>74.71</td>
+ <td>42.75</td>
+ <td>56.42</td>
+ </tr>
+ <tr>
+ <td>GPTQ</td>
+ <td>31.16</td>
+ <td>72.40</td>
+ <td>55.85</td>
+ <td>70.09</td>
+ <td>78.13</td>
+ <td><b>22.28</td>
+ <td>30.40</td>
+ <td>74.65</td>
+ <td>64.26</td>
+ <td>74.20</td>
+ <td>40.19</td>
+ <td>55.78</td>
+ </tr>
+ <tr>
+ <td>AWQ</td>
+ <td><b>33.42</td>
+ <td><b>72.95</td>
+ <td>56.30</td>
+ <td>68.75</td>
+ <td>77.97</td>
+ <td>21.42</td>
+ <td>32.80</td>
+ <td>74.89</td>
+ <td>62.09</td>
+ <td>75.00</td>
+ <td>41.21</td>
+ <td>56.07</td>
+ </tr>
+ <tr>
+ <td>Ours</td>
+ <td>32.15</td>
+ <td>72.85</td>
+ <td><b>56.45</td>
+ <td><b>70.17</td>
+ <td><b>78.51</td>
+ <td><b>22.28</td>
+ <td>32.80</td>
+ <td><b>75.14</td>
+ <td><b>67.87</td>
+ <td><b>75.13</td>
+ <td><b>41.89</td>
+ <td><b>56.84</td>
+ </tr>
+ <tr>
+ <td rowspan="5">V1-13B</td>
+ <td>FP16</td>
+ <td>44.21</td>
+ <td>76.21</td>
+ <td>59.92</td>
+ <td>72.77</td>
+ <td>79.16</td>
+ <td>25.70</td>
+ <td>33.20</td>
+ <td>77.89</td>
+ <td>70.76</td>
+ <td>77.40</td>
+ <td>46.42</td>
+ <td>60.33</td>
+ </tr>
+ <tr>
+ <td>RTN</td>
+ <td><b>42.71</td>
+ <td>75.26</td>
+ <td>59.30</td>
+ <td>72.53</td>
+ <td><b>79.54</td>
+ <td>25.95</td>
+ <td>32.60</td>
+ <td>76.76</td>
+ <td>65.34</td>
+ <td><b>76.98</td>
+ <td>45.82</td>
+ <td>59.34</td>
+ </tr>
+ <tr>
+ <td>GPTQ*</td>
+ <td>42.65</td>
+ <td>75.41</td>
+ <td>59.51</td>
+ <td>72.93</td>
+ <td>79.33</td>
+ <td>24.97</td>
+ <td>32.40</td>
+ <td>77.49</td>
+ <td>68.23</td>
+ <td>76.89</td>
+ <td>45.56</td>
+ <td>59.58</td>
+ </tr>
+ <tr>
+ <td>AWQ</td>
+ <td>42.66</td>
+ <td>75.76</td>
+ <td>59.50</td>
+ <td>72.77</td>
+ <td>78.89</td>
+ <td><b>26.56</td>
+ <td><b>33.60</td>
+ <td>77.46</td>
+ <td>68.59</td>
+ <td>76.94</td>
+ <td>45.48</td>
+ <td>59.84</td>
+ </tr>
+ <tr>
+ <td>Ours</td>
+ <td>42.27</td>
+ <td><b>76.17</td>
+ <td><b>59.53</td>
+ <td><b>73.56</td>
+ <td>79.33</td>
+ <td>25.70</td>
+ <td>32.80</td>
+ <td><b>78.20</td>
+ <td><b>70.04</td>
+ <td>76.94</td>
+ <td><b>46.25</td>
+ <td><b>60.07</td>
+ </tr>
+ <tr>
+ <td rowspan="5">V1-30B</td>
+ <td>FP16</td>
+ <td>55.14</td>
+ <td>77.55</td>
+ <td>63.33</td>
+ <td>75.85</td>
+ <td>81.12</td>
+ <td>28.27</td>
+ <td>36.00</td>
+ <td>82.78</td>
+ <td>66.79</td>
+ <td>80.39</td>
+ <td>52.90</td>
+ <td>63.65</td>
+ </tr>
+ <tr>
+ <td>RTN</td>
+ <td>54.24</td>
+ <td>77.02</td>
+ <td>62.90</td>
+ <td>74.35</td>
+ <td><b>80.52</td>
+ <td>27.29</td>
+ <td>34.20</td>
+ <td>81.96</td>
+ <td>67.15</td>
+ <td>80.89</td>
+ <td>52.05</td>
+ <td>62.96</td>
+ </tr>
+ <tr>
+ <td>GPTQ</td>
+ <td>54.20</td>
+ <td>77.41</td>
+ <td>62.79</td>
+ <td>75.14</td>
+ <td>80.41</td>
+ <td>27.54</td>
+ <td>34.60</td>
+ <td>81.93</td>
+ <td><b>67.51</td>
+ <td>80.05</td>
+ <td>50.51</td>
+ <td>62.92</td>
+ </tr>
+ <tr>
+ <td>AWQ</td>
+ <td><b>55.14</td>
+ <td>77.49</td>
+ <td><b>63.08</td>
+ <td><b>75.77</td>
+ <td><b>80.52</td>
+ <td>27.29</td>
+ <td>34.20</td>
+ <td><b>82.87</td>
+ <td>67.15</td>
+ <td><b>80.43</td>
+ <td><b>52.90</td>
+ <td><b>63.35</td>
+ </tr>
+ <tr>
+ <td>Ours</td>
+ <td>54.68</td>
+ <td><b>77.90</td>
+ <td>62.93</td>
+ <td>74.82</td>
+ <td>80.47</td>
+ <td><b>28.15</td>
+ <td><b>35.80</td>
+ <td>82.39</td>
+ <td>66.79</td>
+ <td>80.13</td>
+ <td>51.11</td>
+ <td>63.20</td>
+ </tr>
+ <tr>
+ <td rowspan="5">V1-65B</td>
+ <td>FP16</td>
+ <td>59.79</td>
+ <td>79.12</td>
+ <td>64.53</td>
+ <td>77.35</td>
+ <td>81.23</td>
+ <td>27.91</td>
+ <td>38.00</td>
+ <td>84.86</td>
+ <td>69.68</td>
+ <td>81.36</td>
+ <td>52.82</td>
+ <td>65.15</td>
+ </tr>
+ <tr>
+ <td>RTN</td>
+ <td>59.53</td>
+ <td><b>79.51</td>
+ <td>64.63</td>
+ <td><b>77.35</td>
+ <td>80.96</td>
+ <td>27.91</td>
+ <td>38.40</td>
+ <td><b>84.43</td>
+ <td><b>71.48</td>
+ <td>81.48</td>
+ <td>52.22</td>
+ <td><b>65.26</td>
+ </tr>
+ <tr>
+ <td>GPTQ*</td>
+ <td><b>60.47</td>
+ <td>78.79</td>
+ <td>64.45</td>
+ <td>76.24</td>
+ <td>81.18</td>
+ <td>28.03</td>
+ <td>37.40</td>
+ <td>83.85</td>
+ <td>68.95</td>
+ <td><b>81.57</td>
+ <td><b>53.07</td>
+ <td>64.91</td>
+ </tr>
+ <tr>
+ <td>AWQ</td>
+ <td>59.45</td>
+ <td>79.31</td>
+ <td><b>64.67</td>
+ <td>76.72</td>
+ <td><b>81.56</td>
+ <td><b>28.15</td>
+ <td>38.00</td>
+ <td><b>84.43</td>
+ <td>71.12</td>
+ <td>81.10</td>
+ <td>52.13</td>
+ <td>65.15</td>
+ </tr>
+ <tr>
+ <td>Ours</td>
+ <td>58.93</td>
+ <td>79.22</td>
+ <td>64.48</td>
+ <td>77.03</td>
+ <td>81.28</td>
+ <td>27.91</td>
+ <td><b>38.60</td>
+ <td>84.31</td>
+ <td>70.76</td>
+ <td>81.19</td>
+ <td>52.22</td>
+ <td>65.08</td>
+ </tr>
+</table>
+
+</br>
+
+### 3. Accuracies $\uparrow$ across 11 tasks(0-shot) of LLaMA and Mistral models at W3G128.
+
+<table border="1">
+ <tr>
+ <td></td>
+ <td></td>
+ <td>Mmlu</td>
+ <td>Lamb.</td>
+ <td>Hella.</td>
+ <td>Wino.</td>
+ <td>Piqa</td>
+ <td>Truth.</td>
+ <td>Open.</td>
+ <td>Boolq</td>
+ <td>RTE</td>
+ <td>ARC-e</td>
+ <td>ARC-c.</td>
+ <td>Avg.</td>
+ </tr>
+ <tr>
+ <td rowspan="5">Mistral-7B</td>
+ <td>FP16</td>
+ <td>61.35</td>
+ <td>75.68</td>
+ <td>61.27</td>
+ <td>74.03</td>
+ <td>80.79</td>
+ <td>28.03</td>
+ <td>32.80</td>
+ <td>83.67</td>
+ <td>67.51</td>
+ <td>80.81</td>
+ <td>50.34</td>
+ <td>63.30</td>
+ </tr>
+ <tr>
+ <td>RTN</td>
+ <td>53.49</td>
+ <td>68.74</td>
+ <td>58.12</td>
+ <td>68.27</td>
+ <td>79.33</td>
+ <td>24.60</td>
+ <td>29.60</td>
+ <td>79.97</td>
+ <td>57.40</td>
+ <td>76.89</td>
+ <td>43.77</td>
+ <td>58.20</td>
+ </tr>
+ <tr>
+ <td>GPTQ</td>
+ <td>55.84</td>
+ <td>73.04</td>
+ <td>57.61</td>
+ <td>70.24</td>
+ <td>78.67</td>
+ <td>24.85</td>
+ <td>30.80</td>
+ <td>81.44</td>
+ <td><b>63.54</td>
+ <td>77.27</td>
+ <td>45.65</td>
+ <td>59.91</td>
+ </tr>
+ <tr>
+ <td>AWQ</td>
+ <td>55.61</td>
+ <td><b>73.69</td>
+ <td>57.86</td>
+ <td>71.27</td>
+ <td><b>79.82</td>
+ <td><b>26.07</td>
+ <td>29.00</td>
+ <td>81.10</td>
+ <td>59.21</td>
+ <td><b>79.00</td>
+ <td><b>46.93</td>
+ <td>59.96</td>
+ </tr>
+ <tr>
+ <td>Ours</td>
+ <td><b>57.54</td>
+ <td>73.01</td>
+ <td><b>59.60</td>
+ <td><b>72.85</td>
+ <td>79.54</td>
+ <td>25.70</td>
+ <td><b>31.60</td>
+ <td><b>81.74</td>
+ <td>58.12</td>
+ <td>78.70</td>
+ <td>46.33</td>
+ <td><b>60.43</td>
+ </tr>
+ <tr>
+ <td rowspan="5">V2-7B</td>
+ <td>FP16</td>
+ <td>42.69</td>
+ <td>73.90</td>
+ <td>57.15</td>
+ <td>68.90</td>
+ <td>78.07</td>
+ <td>25.21</td>
+ <td>31.40</td>
+ <td>77.74</td>
+ <td>62.82</td>
+ <td>76.35</td>
+ <td>43.52</td>
+ <td>57.98</td>
+ </tr>
+ <tr>
+ <td>RTN</td>
+ <td>34.22</td>
+ <td>65.96</td>
+ <td>54.90</td>
+ <td>67.56</td>
+ <td>76.28</td>
+ <td>24.48</td>
+ <td>30.80</td>
+ <td>71.68</td>
+ <td>54.51</td>
+ <td>72.98</td>
+ <td>38.57</td>
+ <td>53.81</td>
+ </tr>
+ <tr>
+ <td>GPTQ</td>
+ <td>36.11</td>
+ <td>69.61</td>
+ <td>53.66</td>
+ <td><b>68.59</td>
+ <td>76.01</td>
+ <td>21.91</td>
+ <td>27.80</td>
+ <td>73.43</td>
+ <td>54.51</td>
+ <td>73.74</td>
+ <td>40.19</td>
+ <td>54.14</td>
+ </tr>
+ <tr>
+ <td>AWQ</td>
+ <td>35.82</td>
+ <td>69.90</td>
+ <td>54.98</td>
+ <td>67.40</td>
+ <td>76.01</td>
+ <td>25.21</td>
+ <td>29.80</td>
+ <td>74.68</td>
+ <td>57.76</td>
+ <td>74.07</td>
+ <td>41.64</td>
+ <td>55.21</td>
+ </tr>
+ <tr>
+ <td>Ours</td>
+ <td><b>40.13</td>
+ <td><b>71.01</td>
+ <td><b>55.33</td>
+ <td>68.27</td>
+ <td><b>76.82</td>
+ <td><b>25.34</td>
+ <td><b>32.80</td>
+ <td><b>75.32</td>
+ <td><b>60.29</td>
+ <td><b>75.25</td>
+ <td><b>42.92</td>
+ <td><b>56.68</td>
+ </tr>
+ <tr>
+ <td rowspan="5">V2-13B</td>
+ <td>FP16</td>
+ <td>52.86</td>
+ <td>76.77</td>
+ <td>60.04</td>
+ <td>72.14</td>
+ <td>79.05</td>
+ <td>25.95</td>
+ <td>35.20</td>
+ <td>80.55</td>
+ <td>65.34</td>
+ <td>79.38</td>
+ <td>48.38</td>
+ <td>61.42</td>
+ </tr>
+ <tr>
+ <td>RTN</td>
+ <td>48.01</td>
+ <td>72.33</td>
+ <td>57.74</td>
+ <td>70.72</td>
+ <td>78.07</td>
+ <td><b>25.21</td>
+ <td>32.00</td>
+ <td>77.28</td>
+ <td>60.65</td>
+ <td>77.69</td>
+ <td>44.62</td>
+ <td>58.57</td>
+ </tr>
+ <tr>
+ <td>GPTQ</td>
+ <td>49.56</td>
+ <td><b>75.24</td>
+ <td>57.83</td>
+ <td>70.88</td>
+ <td><b>78.56</td>
+ <td>24.97</td>
+ <td>33.40</td>
+ <td>78.44</td>
+ <td><b>62.82</td>
+ <td>77.99</td>
+ <td>45.65</td>
+ <td><b>59.58</td>
+ </tr>
+ <tr>
+ <td>AWQ</td>
+ <td><b>49.77</td>
+ <td>75.22</td>
+ <td>58.58</td>
+ <td><b>71.82</td>
+ <td>77.75</td>
+ <td>24.11</td>
+ <td><b>34.20</td>
+ <td><b>79.97</td>
+ <td>53.43</td>
+ <td>77.95</td>
+ <td>44.62</td>
+ <td>58.86</td>
+ </tr>
+ <tr>
+ <td>Ours</td>
+ <td>49.64</td>
+ <td>75.20</td>
+ <td><b>59.11</td>
+ <td>71.59</td>
+ <td>78.29</td>
+ <td>24.85</td>
+ <td><b>34.20</td>
+ <td>78.47</td>
+ <td>58.12</td>
+ <td><b>78.58</td>
+ <td><b>45.82</td>
+ <td>59.44</td>
+ </tr>
+ <tr>
+ <td rowspan="5">V2-70B</td>
+ <td>FP16</td>
+ <td>66.23</td>
+ <td>79.64</td>
+ <td>64.77</td>
+ <td>77.98</td>
+ <td>82.15</td>
+ <td>30.60</td>
+ <td>37.20</td>
+ <td>83.70</td>
+ <td>67.87</td>
+ <td>82.70</td>
+ <td>54.44</td>
+ <td>66.12</td>
+ </tr>
+ <tr>
+ <td>RTN</td>
+ <td>61.15</td>
+ <td>77.95</td>
+ <td>61.98</td>
+ <td><b>77.90</td>
+ <td>80.79</td>
+ <td>29.74</td>
+ <td>36.00</td>
+ <td>81.28</td>
+ <td>64.62</td>
+ <td>81.10</td>
+ <td>52.39</td>
+ <td>64.08</td>
+ </tr>
+ <tr>
+ <td>GPTQ</td>
+ <td>63.15</td>
+ <td>79.06</td>
+ <td>62.94</td>
+ <td>77.66</td>
+ <td>81.45</td>
+ <td>30.72</td>
+ <td>36.20</td>
+ <td>81.53</td>
+ <td>67.87</td>
+ <td>81.65</td>
+ <td><b>53.67</td>
+ <td>65.08</td>
+ </tr>
+ <tr>
+ <td>AWQ</td>
+ <td>64.09</td>
+ <td><b>79.47</td>
+ <td>63.75</td>
+ <td>76.48</td>
+ <td><b>81.77</td>
+ <td>29.74</td>
+ <td><b>37.20</td>
+ <td><b>82.69</td>
+ <td>66.06</td>
+ <td>81.40</td>
+ <td><b>53.67</td>
+ <td>65.12</td>
+ </tr>
+ <tr>
+ <td>Ours</td>
+ <td><b>64.94</td>
+ <td>78.89</td>
+ <td><b>63.83</td>
+ <td>76.56</td>
+ <td>81.50</td>
+ <td><b>31.21</td>
+ <td><b>37.20</td>
+ <td>81.41</td>
+ <td><b>68.59</td>
+ <td><b>81.73</td>
+ <td>52.56</td>
+ <td><b>65.31</td>
+ </tr>
+ <tr>
+ <td rowspan="5">V1-7B</td>
+ <td>FP16</td>
+ <td>32.74</td>
+ <td>73.53</td>
+ <td>56.94</td>
+ <td>70.01</td>
+ <td>78.67</td>
+ <td>22.03</td>
+ <td>34.60</td>
+ <td>75.08</td>
+ <td>66.43</td>
+ <td>75.25</td>
+ <td>41.81</td>
+ <td>57.01</td>
+ </tr>
+ <tr>
+ <td>RTN</td>
+ <td>28.00</td>
+ <td>67.67</td>
+ <td>53.43</td>
+ <td>66.38</td>
+ <td>76.50</td>
+ <td>21.42</td>
+ <td>31.20</td>
+ <td>72.72</td>
+ <td>59.21</td>
+ <td>70.92</td>
+ <td>38.31</td>
+ <td>53.25</td>
+ </tr>
+ <tr>
+ <td>GPTQ</td>
+ <td>30.16</td>
+ <td>66.31</td>
+ <td>53.92</td>
+ <td>67.48</td>
+ <td>76.82</td>
+ <td>21.42</td>
+ <td>29.60</td>
+ <td>71.31</td>
+ <td>59.21</td>
+ <td>72.22</td>
+ <td>38.74</td>
+ <td>53.38</td>
+ </tr>
+ <tr>
+ <td>AWQ</td>
+ <td><b>30.33</td>
+ <td>70.19</td>
+ <td>54.53</td>
+ <td>68.98</td>
+ <td>76.71</td>
+ <td>20.81</td>
+ <td>31.60</td>
+ <td><b>74.68</td>
+ <td><b>64.62</td>
+ <td>73.23</td>
+ <td>38.91</td>
+ <td><b>54.96</td>
+ </tr>
+ <tr>
+ <td>Ours</td>
+ <td>25.85</td>
+ <td><b>70.95</td>
+ <td><b>55.45</td>
+ <td><b>69.69</td>
+ <td><b>77.37</td>
+ <td><b>21.66</td>
+ <td><b>32.00</td>
+ <td>73.88</td>
+ <td>60.29</td>
+ <td><b>73.48</td>
+ <td><b>39.33</td>
+ <td>54.54</td>
+ </tr>
+ <tr>
+ <td rowspan="5">V1-13B</td>
+ <td>FP16</td>
+ <td>44.21</td>
+ <td>76.21</td>
+ <td>59.92</td>
+ <td>72.77</td>
+ <td>79.16</td>
+ <td>25.70</td>
+ <td>33.20</td>
+ <td>77.89</td>
+ <td>70.76</td>
+ <td>77.40</td>
+ <td>46.42</td>
+ <td>60.33</td>
+ </tr>
+ <tr>
+ <td>RTN</td>
+ <td>34.87</td>
+ <td>69.65</td>
+ <td>57.25</td>
+ <td>70.48</td>
+ <td>77.31</td>
+ <td><b>26.93</td>
+ <td>32.00</td>
+ <td>71.44</td>
+ <td>62.82</td>
+ <td>75.63</td>
+ <td>43.94</td>
+ <td>56.57</td>
+ </tr>
+ <tr>
+ <td>GPTQ</td>
+ <td>35.51</td>
+ <td>73.08</td>
+ <td>57.89</td>
+ <td>70.80</td>
+ <td>77.37</td>
+ <td>24.48</td>
+ <td>31.40</td>
+ <td><b>77.52</td>
+ <td>62.82</td>
+ <td>74.41</td>
+ <td>43.26</td>
+ <td>57.14</td>
+ </tr>
+ <tr>
+ <td>AWQ</td>
+ <td><b>40.53</td>
+ <td>73.94</td>
+ <td>57.89</td>
+ <td>69.53</td>
+ <td><b>78.94</td>
+ <td>26.68</td>
+ <td>33.40</td>
+ <td>74.83</td>
+ <td><b>65.34</td>
+ <td>75.93</td>
+ <td>45.05</td>
+ <td>58.37</td>
+ </tr>
+ <tr>
+ <td>Ours</td>
+ <td>39.16</td>
+ <td><b>75.22</td>
+ <td><b>58.64</td>
+ <td><b>71.59</td>
+ <td><b>78.94</td>
+ <td>25.95</td>
+ <td><b>35.20</td>
+ <td>76.30</td>
+ <td><b>65.34</td>
+ <td><b>76.52</td>
+ <td><b>45.39</td>
+ <td><b>58.93</td>
+ </tr>
+ <tr>
+ <td rowspan="5">V1-30B</td>
+ <td>FP16</td>
+ <td>55.14</td>
+ <td>77.55</td>
+ <td>63.33</td>
+ <td>75.85</td>
+ <td>81.12</td>
+ <td>28.27</td>
+ <td>36.00</td>
+ <td>82.78</td>
+ <td>66.79</td>
+ <td>80.39</td>
+ <td>52.90</td>
+ <td>63.65</td>
+ </tr>
+ <tr>
+ <td>RTN</td>
+ <td>52.41</td>
+ <td>75.08</td>
+ <td>61.45</td>
+ <td>74.27</td>
+ <td>79.87</td>
+ <td>25.95</td>
+ <td>33.00</td>
+ <td>81.38</td>
+ <td>65.34</td>
+ <td>79.12</td>
+ <td>48.89</td>
+ <td>61.52</td>
+ </tr>
+ <tr>
+ <td>GPTQ</td>
+ <td>51.39</td>
+ <td>74.97</td>
+ <td>60.35</td>
+ <td><b>75.30</td>
+ <td>79.60</td>
+ <td>26.93</td>
+ <td>34.80</td>
+ <td><b>82.75</td>
+ <td>64.62</td>
+ <td>78.11</td>
+ <td>48.46</td>
+ <td>61.57</td>
+ </tr>
+ <tr>
+ <td>AWQ</td>
+ <td>53.84</td>
+ <td>76.71</td>
+ <td>61.94</td>
+ <td>75.14</td>
+ <td>80.03</td>
+ <td>25.34</td>
+ <td>34.40</td>
+ <td>81.90</td>
+ <td>67.15</td>
+ <td><b>79.59</td>
+ <td><b>50.77</td>
+ <td>62.44</td>
+ </tr>
+ <tr>
+ <td>Ours</td>
+ <td><b>54.39</td>
+ <td><b>77.49</td>
+ <td><b>62.13</td>
+ <td>74.03</td>
+ <td><b>80.47</td>
+ <td><b>27.30</td>
+ <td><b>35.00</td>
+ <td>79.76</td>
+ <td><b>68.59</td>
+ <td>79.46</td>
+ <td>48.98</td>
+ <td><b>62.51</td>
+ </tr>
+ <tr>
+ <td rowspan="5">V1-65B</td>
+ <td>FP16</td>
+ <td>59.79</td>
+ <td>79.12</td>
+ <td>64.53</td>
+ <td>77.35</td>
+ <td>81.23</td>
+ <td>27.91</td>
+ <td>38.00</td>
+ <td>84.86</td>
+ <td>69.68</td>
+ <td>81.36</td>
+ <td>52.82</td>
+ <td>65.15</td>
+ </tr>
+ <tr>
+ <td>RTN</td>
+ <td>57.47</td>
+ <td>77.43</td>
+ <td>63.23</td>
+ <td>75.93</td>
+ <td>80.41</td>
+ <td>28.64</td>
+ <td><b>38.40</td>
+ <td>82.69</td>
+ <td>66.43</td>
+ <td>80.22</td>
+ <td><b>51.19</td>
+ <td>63.82</td>
+ </tr>
+ <tr>
+ <td>GPTQ*</td>
+ <td>57.92</td>
+ <td><b>78.69</td>
+ <td>62.98</td>
+ <td><b>76.87</td>
+ <td>80.63</td>
+ <td>27.66</td>
+ <td>37.60</td>
+ <td>84.16</td>
+ <td>68.95</td>
+ <td>80.89</td>
+ <td><b>51.19</td>
+ <td>64.32</td>
+ </tr>
+ <tr>
+ <td>AWQ</td>
+ <td><b>58.87</td>
+ <td>77.94</td>
+ <td><b>63.77</td>
+ <td>75.37</td>
+ <td><b>80.96</td>
+ <td>27.66</td>
+ <td>36.80</td>
+ <td><b>85.02</td>
+ <td><b>71.12</td>
+ <td><b>81.10</td>
+ <td>50.34</td>
+ <td>64.45</td>
+ </tr>
+ <tr>
+ <td>Ours</td>
+ <td>58.30</td>
+ <td>78.11</td>
+ <td>63.60</td>
+ <td>76.56</td>
+ <td>80.85</td>
+ <td><b>29.50</td>
+ <td>37.80</td>
+ <td>84.80</td>
+ <td>70.04</td>
+ <td>80.22</td>
+ <td>50.68</td>
+ <td><b>64.59</td>
+ </tr>
+</table>
+
+</br>
+
+### 4. Accuracies $\uparrow$ across 11 tasks(0-shot) of LLaMA and Mistral models at W2G128.
+
+<table border="1">
+ <tr>
+ <td></td>
+ <td></td>
+ <td>Mmlu</td>
+ <td>Lamb.</td>
+ <td>Hella.</td>
+ <td>Wino.</td>
+ <td>Piqa</td>
+ <td>Truth.</td>
+ <td>Open.</td>
+ <td>Boolq</td>
+ <td>RTE</td>
+ <td>ARC-e</td>
+ <td>ARC-c.</td>
+ <td>Avg.</td>
+ </tr>
+ <tr>
+ <td rowspan="5">Mistral-7B</td>
+ <td>FP16</td>
+ <td>61.35</td>
+ <td>75.68</td>
+ <td>61.27</td>
+ <td>74.03</td>
+ <td>80.79</td>
+ <td>28.03</td>
+ <td>32.80</td>
+ <td>83.67</td>
+ <td>67.51</td>
+ <td>80.81</td>
+ <td>50.34</td>
+ <td>63.30</td>
+ </tr>
+ <tr>
+ <td>RTN</td>
+ <td>23.45</td>
+ <td>0.14</td>
+ <td>27.43</td>
+ <td>49.64</td>
+ <td>54.30</td>
+ <td>24.24</td>
+ <td>15.20</td>
+ <td>38.69</td>
+ <td>51.99</td>
+ <td>29.08</td>
+ <td>21.59</td>
+ <td>30.52</td>
+ </tr>
+ <tr>
+ <td>GPTQ</td>
+ <td>25.23</td>
+ <td>30.47</td>
+ <td>38.28</td>
+ <td>53.83</td>
+ <td>64.91</td>
+ <td>24.11</td>
+ <td>17.40</td>
+ <td>58.29</td>
+ <td>50.90</td>
+ <td>47.77</td>
+ <td>24.57</td>
+ <td>39.61</td>
+ </tr>
+ <tr>
+ <td>AWQ</td>
+ <td>25.38</td>
+ <td>0.00</td>
+ <td>25.71</td>
+ <td>52.01</td>
+ <td>51.58</td>
+ <td>23.99</td>
+ <td>17.60</td>
+ <td>37.83</td>
+ <td>47.29</td>
+ <td>26.98</td>
+ <td>22.27</td>
+ <td>30.06</td>
+ </tr>
+ <tr>
+ <td>Ours</td>
+ <td><b>40.46</td>
+ <td><b>58.61</td>
+ <td><b>50.87</td>
+ <td><b>62.90</td>
+ <td><b>75.84</td>
+ <td><b>24.85</td>
+ <td><b>22.80</td>
+ <td><b>78.56</td>
+ <td><b>57.04</td>
+ <td><b>70.88</td>
+ <td><b>37.03</td>
+ <td><b>52.71 </td>
+ </tr>
+ <tr>
+ <td rowspan="5">V2-7B</td>
+ <td>FP16</td>
+ <td>42.69</td>
+ <td>73.90</td>
+ <td>57.15</td>
+ <td>68.90</td>
+ <td>78.07</td>
+ <td>25.21</td>
+ <td>31.40</td>
+ <td>77.74</td>
+ <td>62.82</td>
+ <td>76.35</td>
+ <td>43.52</td>
+ <td>57.98</td>
+ </tr>
+ <tr>
+ <td>RTN</td>
+ <td>23.98</td>
+ <td>0.02</td>
+ <td>26.04</td>
+ <td>49.49</td>
+ <td>52.50</td>
+ <td>24.85</td>
+ <td>15.20</td>
+ <td>41.01</td>
+ <td>49.10</td>
+ <td>27.48</td>
+ <td>19.71</td>
+ <td>29.94</td>
+ </tr>
+ <tr>
+ <td>GPTQ</td>
+ <td>23.65</td>
+ <td>11.72</td>
+ <td>32.59</td>
+ <td>55.17</td>
+ <td>58.32</td>
+ <td><b>25.95</td>
+ <td>15.80</td>
+ <td>52.14</td>
+ <td>51.99</td>
+ <td>40.45</td>
+ <td>21.25</td>
+ <td>35.37</td>
+ </tr>
+ <tr>
+ <td>AWQ</td>
+ <td>25.38</td>
+ <td>0.00</td>
+ <td>25.69</td>
+ <td>49.96</td>
+ <td>52.34</td>
+ <td>23.75</td>
+ <td>17.80</td>
+ <td>37.83</td>
+ <td>52.71</td>
+ <td>24.62</td>
+ <td>21.08</td>
+ <td>30.10</td>
+ </tr>
+ <tr>
+ <td>Ours</td>
+ <td><b>27.20</td>
+ <td><b>55.25</td>
+ <td><b>47.35</td>
+ <td><b>61.01</td>
+ <td><b>72.96</td>
+ <td>24.85</td>
+ <td><b>25.60</td>
+ <td><b>68.07</td>
+ <td><b>54.51</td>
+ <td><b>65.99</td>
+ <td><b>32.25</td>
+ <td><b>48.64</td>
+ </tr>
+ <tr>
+ <td rowspan="5">V2-13B</td>
+ <td>FP16</td>
+ <td>52.86</td>
+ <td>76.77</td>
+ <td>60.04</td>
+ <td>72.14</td>
+ <td>79.05</td>
+ <td>25.95</td>
+ <td>35.20</td>
+ <td>80.55</td>
+ <td>65.34</td>
+ <td>79.38</td>
+ <td>48.38</td>
+ <td>61.42</td>
+ </tr>
+ <tr>
+ <td>RTN</td>
+ <td>23.77</td>
+ <td>7.47</td>
+ <td>33.08</td>
+ <td>49.01</td>
+ <td>57.94</td>
+ <td><b>26.19</td>
+ <td>16.00</td>
+ <td>47.74</td>
+ <td>53.43</td>
+ <td>32.03</td>
+ <td>21.93</td>
+ <td>33.51</td>
+ </tr>
+ <tr>
+ <td>GPTQ</td>
+ <td>24.69</td>
+ <td>45.20</td>
+ <td>41.06</td>
+ <td>55.80</td>
+ <td>67.08</td>
+ <td>23.26</td>
+ <td>19.80</td>
+ <td>54.40</td>
+ <td>52.35</td>
+ <td>55.60</td>
+ <td>27.82</td>
+ <td>42.46</td>
+ </tr>
+ <tr>
+ <td>AWQ</td>
+ <td>27.04</td>
+ <td>0.00</td>
+ <td>25.80</td>
+ <td>51.85</td>
+ <td>52.99</td>
+ <td>23.62</td>
+ <td>13.60</td>
+ <td>62.17</td>
+ <td>47.29</td>
+ <td>26.22</td>
+ <td>23.12</td>
+ <td>32.16</td>
+ </tr>
+ <tr>
+ <td>Ours</td>
+ <td><b>34.33</td>
+ <td><b>63.92</td>
+ <td><b>53.35</td>
+ <td><b>64.33</td>
+ <td><b>76.17</td>
+ <td>25.70</td>
+ <td><b>26.00</td>
+ <td><b>72.75</td>
+ <td><b>61.73</td>
+ <td><b>71.17</td>
+ <td><b>38.57</td>
+ <td><b>53.46</td>
+ </tr>
+ <tr>
+ <td rowspan="5">V2-70B</td>
+ <td>FP16</td>
+ <td>66.23</td>
+ <td>79.64</td>
+ <td>64.77</td>
+ <td>77.98</td>
+ <td>82.15</td>
+ <td>30.60</td>
+ <td>37.20</td>
+ <td>83.70</td>
+ <td>67.87</td>
+ <td>82.70</td>
+ <td>54.44</td>
+ <td>66.12</td>
+ </tr>
+ <tr>
+ <td>RTN</td>
+ <td>24.20</td>
+ <td>20.18</td>
+ <td>40.88</td>
+ <td>54.85</td>
+ <td>63.87</td>
+ <td>24.11</td>
+ <td>17.60</td>
+ <td>43.06</td>
+ <td>53.07</td>
+ <td>50.51</td>
+ <td>27.22</td>
+ <td>38.14</td>
+ </tr>
+ <tr>
+ <td>GPTQ</td>
+ <td>23.12</td>
+ <td>0.00</td>
+ <td>25.04</td>
+ <td>49.57</td>
+ <td>49.51</td>
+ <td>0.00</td>
+ <td>27.60</td>
+ <td>37.83</td>
+ <td>52.71</td>
+ <td>25.08</td>
+ <td>22.70</td>
+ <td>28.47</td>
+ </tr>
+ <tr>
+ <td>AWQ</td>
+ <td>24.46</td>
+ <td>0.00</td>
+ <td>25.46</td>
+ <td>51.38</td>
+ <td>52.50</td>
+ <td>23.50</td>
+ <td>14.20</td>
+ <td>62.17</td>
+ <td>52.71</td>
+ <td>25.76</td>
+ <td>22.35</td>
+ <td>32.23</td>
+ </tr>
+ <tr>
+ <td>Ours</td>
+ <td><b>54.04</td>
+ <td><b>72.97</td>
+ <td><b>59.65</td>
+ <td><b>74.90</td>
+ <td><b>79.00</td>
+ <td><b>29.01</td>
+ <td><b>34.80 </td>
+ <td><b>79.63 </td>
+ <td><b>69.68</td>
+ <td><b>78.37 </td>
+ <td><b> 46.59</td>
+ <td><b>61.69</td>
+ </tr>
+ <tr>
+ <td rowspan="5">V1-7B</td>
+ <td>FP16</td>
+ <td>32.74</td>
+ <td>73.53</td>
+ <td>56.94</td>
+ <td>70.01</td>
+ <td>78.67</td>
+ <td>22.03</td>
+ <td>34.60</td>
+ <td>75.08</td>
+ <td>66.43</td>
+ <td>75.25</td>
+ <td>41.81</td>
+ <td>57.01</td>
+ </tr>
+ <tr>
+ <td>RTN</td>
+ <td>24.36</td>
+ <td>0.52</td>
+ <td>27.24</td>
+ <td>49.25</td>
+ <td>54.24</td>
+ <td>24.24</td>
+ <td>15.20</td>
+ <td>39.63</td>
+ <td><b>57.40</td>
+ <td>27.86</td>
+ <td>21.84</td>
+ <td>31.07</td>
+ </tr>
+ <tr>
+ <td>GPTQ</td>
+ <td>22.95</td>
+ <td>12.75</td>
+ <td>33.36</td>
+ <td>51.70</td>
+ <td>60.07</td>
+ <td>23.99</td>
+ <td>13.40</td>
+ <td>48.62</td>
+ <td>53.07</td>
+ <td>40.82</td>
+ <td>21.50</td>
+ <td>34.75</td>
+ </tr>
+ <tr>
+ <td>AWQ</td>
+ <td>23.12</td>
+ <td>0.00</td>
+ <td>25.37</td>
+ <td>53.28</td>
+ <td>52.56</td>
+ <td><b>25.21</td>
+ <td>13.80</td>
+ <td>37.83</td>
+ <td>52.71</td>
+ <td>25.63</td>
+ <td>22.53</td>
+ <td>30.18</td>
+ </tr>
+ <tr>
+ <td>Ours</td>
+ <td><b>24.46</td>
+ <td><b>13.53</td>
+ <td><b>42.16</td>
+ <td><b>56.99</td>
+ <td><b>70.02</td>
+ <td>24.60</td>
+ <td><b>25.20</td>
+ <td><b>62.91</td>
+ <td>47.29</td>
+ <td><b>60.90</td>
+ <td><b>31.74</td>
+ <td><b>41.80</td>
+ </tr>
+ <tr>
+ <td rowspan="5">V1-13B</td>
+ <td>FP16</td>
+ <td>44.21</td>
+ <td>76.21</td>
+ <td>59.92</td>
+ <td>72.77</td>
+ <td>79.16</td>
+ <td>25.70</td>
+ <td>33.20</td>
+ <td>77.89</td>
+ <td>70.76</td>
+ <td>77.40</td>
+ <td>46.42</td>
+ <td>60.33</td>
+ </tr>
+ <tr>
+ <td>RTN</td>
+ <td>24.66</td>
+ <td>4.97</td>
+ <td>29.67</td>
+ <td>49.33</td>
+ <td>57.24</td>
+ <td><b>25.58</td>
+ <td>12.40</td>
+ <td>44.10</td>
+ <td>53.79</td>
+ <td>32.07</td>
+ <td>22.01</td>
+ <td>32.35</td>
+ </tr>
+ <tr>
+ <td>GPTQ*</td>
+ <td>26.43</td>
+ <td>40.48</td>
+ <td>39.47</td>
+ <td>58.25</td>
+ <td>66.97</td>
+ <td>23.50</td>
+ <td>18.60</td>
+ <td>52.78</td>
+ <td>50.54</td>
+ <td>51.52</td>
+ <td>25.00</td>
+ <td>41.23</td>
+ </tr>
+ <tr>
+ <td>AWQ</td>
+ <td>27.04</td>
+ <td>0.00</td>
+ <td>25.59</td>
+ <td>50.36</td>
+ <td>53.05</td>
+ <td>24.11</td>
+ <td>15.60</td>
+ <td>62.17</td>
+ <td>47.29</td>
+ <td>25.97</td>
+ <td>23.21</td>
+ <td>32.22</td>
+ </tr>
+ <tr>
+ <td>Ours</td>
+ <td><b>31.87</td>
+ <td><b>59.65</td>
+ <td><b>51.25</td>
+ <td><b>67.64</td>
+ <td><b>76.28</td>
+ <td><b>25.58</td>
+ <td><b>27.80</td>
+ <td><b>69.11</td>
+ <td><b>58.48</td>
+ <td><b>70.71</td>
+ <td><b>37.12</td>
+ <td><b>52.32</td>
+ </tr>
+ <tr>
+ <td rowspan="5">V1-30B</td>
+ <td>FP16</td>
+ <td>55.14</td>
+ <td>77.55</td>
+ <td>63.33</td>
+ <td>75.85</td>
+ <td>81.12</td>
+ <td>28.27</td>
+ <td>36.00</td>
+ <td>82.78</td>
+ <td>66.79</td>
+ <td>80.39</td>
+ <td>52.90</td>
+ <td>63.65</td>
+ </tr>
+ <tr>
+ <td>RTN</td>
+ <td>23.24</td>
+ <td>5.55</td>
+ <td>27.22</td>
+ <td>53.99</td>
+ <td>56.80</td>
+ <td>21.79</td>
+ <td>18.20</td>
+ <td>51.65</td>
+ <td>53.07</td>
+ <td>36.74</td>
+ <td>21.33</td>
+ <td>33.60</td>
+ </tr>
+ <tr>
+ <td>GPTQ</td>
+ <td>30.47</td>
+ <td>49.93</td>
+ <td>45.05</td>
+ <td>61.88</td>
+ <td>68.88</td>
+ <td>23.26</td>
+ <td>22.60</td>
+ <td>68.29</td>
+ <td>51.99</td>
+ <td>60.69</td>
+ <td>30.72</td>
+ <td>46.70</td>
+ </tr>
+ <tr>
+ <td>AWQ</td>
+ <td>27.04</td>
+ <td>0.00</td>
+ <td>25.41</td>
+ <td>50.20</td>
+ <td>52.94</td>
+ <td><b>24.48</td>
+ <td>16.60</td>
+ <td>62.17</td>
+ <td>47.29</td>
+ <td>24.71</td>
+ <td>23.38</td>
+ <td>32.20</td>
+ </tr>
+ <tr>
+ <td>Ours</td>
+ <td><b>40.83</td>
+ <td><b>67.92</td>
+ <td><b>56.73</td>
+ <td><b>68.90</td>
+ <td><b>76.17</td>
+ <td>24.36</td>
+ <td><b>31.60</td>
+ <td><b>75.54</td>
+ <td><b>62.45</td>
+ <td><b>74.92</td>
+ <td><b>42.41</td>
+ <td><b>56.53</td>
+ </tr>
+ <tr>
+ <td rowspan="5">V1-65B</td>
+ <td>FP16</td>
+ <td>59.79</td>
+ <td>79.12</td>
+ <td>64.53</td>
+ <td>77.35</td>
+ <td>81.23</td>
+ <td>27.91</td>
+ <td>38.00</td>
+ <td>84.86</td>
+ <td>69.68</td>
+ <td>81.36</td>
+ <td>52.82</td>
+ <td>65.15</td>
+ </tr>
+ <tr>
+ <td>RTN</td>
+ <td>24.48</td>
+ <td>32.78</td>
+ <td>43.59</td>
+ <td>57.85</td>
+ <td>67.52</td>
+ <td>22.89</td>
+ <td>22.80</td>
+ <td>61.53</td>
+ <td>50.54</td>
+ <td>52.10</td>
+ <td>28.24</td>
+ <td>42.21</td>
+ </tr>
+ <tr>
+ <td>GPTQ*</td>
+ <td>37.06</td>
+ <td>67.44</td>
+ <td>53.97</td>
+ <td>69.46</td>
+ <td>76.44</td>
+ <td>24.36</td>
+ <td>28.00</td>
+ <td>73.64</td>
+ <td>60.29</td>
+ <td>71.34</td>
+ <td>38.57</td>
+ <td>54.60</td>
+ </tr>
+ <tr>
+ <td>AWQ</td>
+ <td>25.38</td>
+ <td>0.00</td>
+ <td>25.58</td>
+ <td>49.96</td>
+ <td>53.10</td>
+ <td>24.24</td>
+ <td>11.00</td>
+ <td>37.83</td>
+ <td>52.71</td>
+ <td>24.96</td>
+ <td>22.44</td>
+ <td>29.75</td>
+ </tr>
+ <tr>
+ <td>Ours</td>
+ <td><b>47.21</td>
+ <td><b>72.07</td>
+ <td><b>60.06</td>
+ <td><b> 73.24</td>
+ <td><b>78.62</td>
+ <td><b>25.46</td>
+ <td><b>34.20</td>
+ <td><b>80.64</td>
+ <td><b>62.82</td>
+ <td><b>77.48</td>
+ <td><b>46.76</td>
+ <td><b>59.87</td>
+ </tr>
+</table>
+
+
diff --git a/docs/W2G128.png b/docs/W2G128.png
deleted file mode 100644
index 1fdb363ad..000000000
Binary files a/docs/W2G128.png and /dev/null differ
diff --git a/docs/W3G128.png b/docs/W3G128.png
deleted file mode 100644
index f0f00e9ec..000000000
Binary files a/docs/W3G128.png and /dev/null differ
diff --git a/docs/W4G-1.png b/docs/W4G-1.png
deleted file mode 100644
index 4180a4445..000000000
Binary files a/docs/W4G-1.png and /dev/null differ
diff --git a/docs/W4G128.png b/docs/W4G128.png
deleted file mode 100644
index 0771853f6..000000000
Binary files a/docs/W4G128.png and /dev/null differ
diff --git a/examples/eval/evaluation.py b/examples/eval/evaluation.py
index a43202120..b0a423993 100644
--- a/examples/eval/evaluation.py
+++ b/examples/eval/evaluation.py
@@ -221,11 +221,16 @@ def eval_model(output_dir=None, model=None, tokenizer=None,
if os.path.exists(output_dir) and not eval_orig_float:
shutil.rmtree(output_dir)
- if (hasattr(model, 'config') and model.config.torch_dtype is torch.bfloat16):
+ if (hasattr(model, 'config') and (model.dtype is torch.bfloat16 or model.config.torch_dtype is torch.bfloat16)):
dtype = 'bfloat16'
pt_dtype = torch.bfloat16
else:
- pt_dtype = torch.float16
+ if str(device) != "cpu":
+ pt_dtype = torch.float16
+ dtype = 'float16'
+ else:
+ pt_dtype = torch.float32
+ dtype = 'float32'
if not eval_orig_float:
model = model.to(pt_dtype)
@@ -372,3 +377,4 @@ if __name__ == "__main__":
print("cost time: ", time.time() - s)
+
diff --git a/examples/main.py b/examples/main.py
index 3ef71b559..f466c8e44 100644
--- a/examples/main.py
+++ b/examples/main.py
@@ -1,6 +1,6 @@
import argparse
import sys
-sys.path.insert(0, './')
+sys.path.insert(0, '../')
from auto_round import (AutoRound,
AutoAdamRound)
@@ -53,7 +53,7 @@ if __name__ == '__main__':
help="whether to use the output of quantized block to tune the next block")
parser.add_argument("--lr", default=None, type=float,
- help="learning rate, if None, it will be set to 1.0/iters automatially")
+ help="learning rate, if None, it will be set to 1.0/iters automatically")
parser.add_argument("--minmax_lr", default=None, type=float,
help="minmax learning rate, if None,it will beset to be the same with lr")
@@ -119,11 +119,7 @@ if __name__ == '__main__':
device_str = f"cuda:{int(args.device)}"
torch_device = torch.device(device_str)
is_glm = bool(re.search("chatglm", model_name.lower()))
- is_llava = bool(re.search("llava", model_name.lower()))
- if is_llava:
- from transformers import LlavaForConditionalGeneration
- model = LlavaForConditionalGeneration.from_pretrained(model_name, low_cpu_mem_usage=True, torch_dtype="auto")
- elif is_glm:
+ if is_glm:
model = AutoModel.from_pretrained(model_name, trust_remote_code=True)
else:
model = AutoModelForCausalLM.from_pretrained(
@@ -169,7 +165,7 @@ if __name__ == '__main__':
scheme = "asym"
if args.sym:
- scheme = "sym"
+ scheme = "sym"
round = AutoRound
if args.adam:
round = AutoAdamRound
@@ -181,18 +177,19 @@ if __name__ == '__main__':
seed=args.seed, gradient_accumulate_steps=args.gradient_accumulate_steps, scale_dtype=args.scale_dtype) ##TODO args pass
model, q_config = autoround.quantize()
model_name = args.model_name.rstrip("/")
- export_dir = args.output_dir + "/compressed_" + model_name.split('/')[-1] + "/"
+ export_dir = args.output_dir + "/" + model_name.split('/')[-1] + "-autoround-int4"
if args.deployment_device == 'cpu':
autoround.export(output_dir=export_dir)
del q_config
elif args.deployment_device == 'gpu':
autoround.export(export_dir, target="auto_gptq", use_triton=True)
+
if args.device != "cpu":
torch.cuda.empty_cache()
model.eval()
- output_dir = args.output_dir + "_" + model_name.split('/')[-1] + f"_w{args.bits}_g{args.group_size}"
+ output_dir = args.output_dir + "/" + model_name.split('/')[-1] + f"-autoround-qdq"# + f"_w{args.bits}_g{args.group_size}"
- excel_name = f"{output_dir}_result.xlsx"
+ excel_name = f"{output_dir}/result.xlsx"
output_dir += "/"
print(excel_name, flush=True)
eval_model(output_dir=output_dir, model=model, tokenizer=tokenizer, tasks=args.tasks, \