From a0df276e976f213f64cf2a6247ba5216acae0309 Mon Sep 17 00:00:00 2001 From: Lesheng Jin Date: Thu, 24 Aug 2023 20:57:31 +0000 Subject: [PATCH 1/3] add whisper and test --- mlc_llm/models/whisper.py | 430 +++++++++++++++++++++++++++++ tests/python/test_model_whisper.py | 168 +++++++++++ 2 files changed, 598 insertions(+) create mode 100644 mlc_llm/models/whisper.py create mode 100644 tests/python/test_model_whisper.py diff --git a/mlc_llm/models/whisper.py b/mlc_llm/models/whisper.py new file mode 100644 index 0000000000..686a996cba --- /dev/null +++ b/mlc_llm/models/whisper.py @@ -0,0 +1,430 @@ +import dataclasses +from typing import Tuple, Optional + +import tvm +from tvm import te, tir +import tvm.relax.frontend.nn as nn +from tvm.relax.frontend.nn import ( + Embedding, + KVCache, + Linear, + LayerNorm, + Conv1D, + Module, + ModuleList, + Parameter, + Tensor, + tensor_expr_op, + permute_dims, + reshape, + squeeze, + matmul, + maximum, + minimum, + softmax, + gelu, + print_, +) + + +@dataclasses.dataclass +class WhisperConfig: + def __init__( + self, + dtype="float32", + vocab_size=51865, + num_mel_bins=80, + encoder_layers=6, + encoder_attention_heads=4, + decoder_layers=6, + decoder_attention_heads=4, + decoder_ffn_dim=1536, + encoder_ffn_dim=1536, + decoder_start_token_id=50257, + d_model=256, + max_source_positions=1500, + max_target_positions=448, + pad_token_id=50256, + bos_token_id=50257, + eos_token_id=50256, + suppress_tokens=None, + begin_suppress_tokens=[220, 50256], + forced_decoder_ids=None, + **kwargs, + ): + self.dtype = dtype + self.vocab_size = vocab_size + self.num_mel_bins = num_mel_bins + self.d_model = d_model + self.encoder_layers = encoder_layers + self.encoder_attention_heads = encoder_attention_heads + self.decoder_layers = decoder_layers + self.decoder_attention_heads = decoder_attention_heads + self.decoder_ffn_dim = decoder_ffn_dim + self.encoder_ffn_dim = encoder_ffn_dim + self.num_hidden_layers = encoder_layers + self.max_source_positions = max_source_positions + self.max_target_positions = max_target_positions + self.pad_token_id = pad_token_id + self.bos_token_id = bos_token_id + self.eos_token_id = eos_token_id + self.decoder_start_token_id = decoder_start_token_id + self.suppress_tokens = suppress_tokens + self.begin_suppress_tokens = begin_suppress_tokens + self.forced_decoder_ids = forced_decoder_ids + self.kwargs = kwargs + + +class WhisperPositionalEmbedding(Module): + def __init__(self, max_seq_len: int, embed_dim: int, dtype: str): + super().__init__() + self.max_seq_len = max_seq_len + self.embed_dim = embed_dim + self.weight = Parameter((max_seq_len, embed_dim), dtype=dtype) + + def forward(self, x: Tensor, offset: tir.Var): + def te_op(x: te.Tensor, embed: te.Tensor, offset: tir.Var): + def compute(i: tir.Var, j: tir.Var, k: tir.Var): + return embed[offset + j, k] + + return tvm.te.compute([*x.shape, embed.shape[-1]], compute, name="position_embedding") + + pos_embed = tensor_expr_op(te_op, "position_embedding", args=[x, self.weight, offset]) + return pos_embed + + +class WhisperAttention(Module): + def __init__( + self, embed_dim: int, num_heads: int, kv_cache_len: int, dtype: str, bias: bool = True + ): + super().__init__() + self.embed_dim = embed_dim + self.num_heads = num_heads + self.head_dim = self.embed_dim // self.num_heads + + if (self.head_dim * num_heads) != self.embed_dim: + raise ValueError( + f"embed_dim must be divisible by num_heads (got `embed_dim`: {self.embed_dim}" + f" and `num_heads`: {num_heads})." + ) + self.scaling = self.head_dim**-0.5 + self.k_proj = Linear(self.embed_dim, self.embed_dim, dtype=dtype, bias=False) + self.v_proj = Linear(self.embed_dim, self.embed_dim, dtype=dtype, bias=bias) + self.q_proj = Linear(self.embed_dim, self.embed_dim, dtype=dtype, bias=bias) + self.out_proj = Linear(self.embed_dim, self.embed_dim, dtype=dtype, bias=bias) + if kv_cache_len > 0: + self.k_cache = KVCache(kv_cache_len, [self.num_heads, self.head_dim]) + self.v_cache = KVCache(kv_cache_len, [self.num_heads, self.head_dim]) + + def forward( + self, + hidden_states: Tensor, + key_value_states: Optional[Tensor] = None, + cached_cross_attn_states: Optional[Tuple[Tensor]] = None, + attention_mask: Optional[Tensor] = None, + total_seq_len: Optional[tir.Var] = None, + ) -> Tuple[Tensor, Optional[Tuple[Tensor]]]: + is_cross_attention = key_value_states is not None or cached_cross_attn_states is not None + + h, d = self.num_heads, self.head_dim + bsz, q_len, _ = hidden_states.shape + assert bsz == 1, "Only support batch size 1 at this moment." + + q = reshape(self.q_proj(hidden_states) * self.scaling, (bsz, q_len, h, d)) + + if is_cross_attention: + # cross attention + if cached_cross_attn_states is None: + # no cache, cross attentions + kv_len = key_value_states.shape[1] + k = reshape(self.k_proj(key_value_states), (bsz, kv_len, h, d)) + v = reshape(self.v_proj(key_value_states), (bsz, kv_len, h, d)) + cached_kv = (k, v) + else: + # reuse cached k,v, cross_attentions + k, v = cached_cross_attn_states + else: + # self attention + k = reshape(self.k_proj(hidden_states), (bsz, q_len, h, d)) + v = reshape(self.v_proj(hidden_states), (bsz, q_len, h, d)) + + if total_seq_len is not None: + # reuse cached k, v, self_attention + self.k_cache.append(squeeze(k, axis=0)) + self.v_cache.append(squeeze(v, axis=0)) + k = reshape(self.k_cache.view(total_seq_len), (bsz, total_seq_len, h, d)) + v = reshape(self.v_cache.view(total_seq_len), (bsz, total_seq_len, h, d)) + else: + # encode self attention, no cache + ... + + q = permute_dims(q, [0, 2, 1, 3]) # [b, h, q_len, d] + k = permute_dims(k, [0, 2, 1, 3]) # [b, h, q_len, d] + v = permute_dims(v, [0, 2, 1, 3]) # [b, h, q_len, d] + + attn_weights = matmul(q, (permute_dims(k, [0, 1, 3, 2]))) # [b, h, q_len, q_len] + + if attention_mask is not None: + attn_weights = attn_weights + attention_mask + + dtype = attn_weights.dtype + attn_weights = attn_weights.maximum(tir.min_value(dtype)) + attn_weights = attn_weights.minimum(tir.max_value(dtype)) + if dtype == "float32": + attn_weights = softmax(attn_weights, axis=-1) + else: + attn_weights = softmax(attn_weights.astype("float32"), axis=-1).astype(dtype) + attn_output = matmul(attn_weights, v) # [b, h, q_len, d] + + attn_output = permute_dims(attn_output, [0, 2, 1, 3]) # [b, q_len, h, d] + attn_output = reshape(attn_output, (bsz, q_len, self.embed_dim)) # [b, q_len, h * d] + + attn_output = self.out_proj(attn_output) + + if is_cross_attention and cached_cross_attn_states is None: + return attn_output, cached_kv + else: + return attn_output, None + + +class EncoderLayer(Module): + def __init__(self, config: WhisperConfig): + super().__init__() + self.embed_dim = config.d_model + self.self_attn = WhisperAttention( + embed_dim=self.embed_dim, + num_heads=config.encoder_attention_heads, + kv_cache_len=0, # no need for kv_cache + dtype=config.dtype, + ) + self.self_attn_layer_norm = LayerNorm(self.embed_dim, dtype=config.dtype) + self.fc1 = Linear(self.embed_dim, config.encoder_ffn_dim, dtype=config.dtype) + self.fc2 = Linear(config.encoder_ffn_dim, self.embed_dim, dtype=config.dtype) + self.final_layer_norm = LayerNorm(self.embed_dim, dtype=config.dtype) + + def forward(self, hidden_states: Tensor, attention_mask: Optional[Tensor] = None) -> Tensor: + residual = hidden_states + hidden_states = self.self_attn_layer_norm(hidden_states) + hidden_states, _ = self.self_attn( + hidden_states=hidden_states, + key_value_states=None, + cached_cross_attn_states=None, + attention_mask=attention_mask, + total_seq_len=None, + ) + hidden_states = residual + hidden_states + + residual = hidden_states + hidden_states = self.final_layer_norm(hidden_states) + hidden_states = gelu(self.fc1(hidden_states)) + hidden_states = self.fc2(hidden_states) + hidden_states = residual + hidden_states + hidden_states = hidden_states.maximum(tir.min_value(hidden_states.dtype)) + hidden_states = hidden_states.minimum(tir.max_value(hidden_states.dtype)) + + return hidden_states + + +class DecoderLayer(Module): + def __init__(self, config: WhisperConfig): + super().__init__() + self.embed_dim = config.d_model + + self.self_attn = WhisperAttention( + embed_dim=self.embed_dim, + num_heads=config.decoder_attention_heads, + kv_cache_len=100, # TODO + dtype=config.dtype, + ) + + self.self_attn_layer_norm = LayerNorm(self.embed_dim, dtype=config.dtype) + self.encoder_attn = WhisperAttention( + self.embed_dim, + config.decoder_attention_heads, + kv_cache_len=100, # TODO + dtype=config.dtype, + ) + self.encoder_attn_layer_norm = LayerNorm(self.embed_dim, dtype=config.dtype) + self.fc1 = Linear(self.embed_dim, config.decoder_ffn_dim, dtype=config.dtype) + self.fc2 = Linear(config.decoder_ffn_dim, self.embed_dim, dtype=config.dtype) + self.final_layer_norm = LayerNorm(self.embed_dim, dtype=config.dtype) + + def forward( + self, + hidden_states: Tensor, + encoder_hidden_states: Tensor, + cached_encoder_hidden_states: Tensor, + total_seq_len: tir.Var, + attention_mask: Optional[Tensor] = None, + encoder_attention_mask: Optional[Tensor] = None, + ) -> Tuple[Tensor, Optional[Tuple[Tensor]]]: + residual = hidden_states + hidden_states = self.self_attn_layer_norm(hidden_states) + + # Self Attention + hidden_states, _ = self.self_attn( + hidden_states=hidden_states, + total_seq_len=total_seq_len, + key_value_states=None, + cached_cross_attn_states=None, + attention_mask=attention_mask, + ) + hidden_states = residual + hidden_states + + # Cross-Attention Block + residual = hidden_states + hidden_states = self.encoder_attn_layer_norm(hidden_states) + hidden_states, cross_attn_key_value = self.encoder_attn( + hidden_states=hidden_states, + total_seq_len=total_seq_len, + key_value_states=encoder_hidden_states, + cached_cross_attn_states=cached_encoder_hidden_states, + attention_mask=encoder_attention_mask, + ) + hidden_states = residual + hidden_states + + # Fully Connected + residual = hidden_states + hidden_states = self.final_layer_norm(hidden_states) + hidden_states = gelu(self.fc1(hidden_states)) + hidden_states = self.fc2(hidden_states) + hidden_states = residual + hidden_states + + if cached_encoder_hidden_states is None: + return hidden_states, cross_attn_key_value + else: + return hidden_states, None + + +class WhisperEncoder(Module): + def __init__(self, config: WhisperConfig): + super().__init__() + + embed_dim = config.d_model + self.num_mel_bins = config.num_mel_bins + self.padding_idx = config.pad_token_id + self.max_source_positions = config.max_source_positions + self.embed_scale = 1.0 + + self.conv1 = Conv1D( + self.num_mel_bins, embed_dim, kernel_size=3, padding=1, dtype=config.dtype + ) + self.conv2 = Conv1D( + embed_dim, embed_dim, kernel_size=3, stride=2, padding=1, dtype=config.dtype + ) + + self.embed_positions = Embedding(self.max_source_positions, embed_dim, dtype=config.dtype) + + self.layers = ModuleList([EncoderLayer(config) for _ in range(config.encoder_layers)]) + self.layer_norm = LayerNorm(config.d_model, dtype=config.dtype) + + def forward(self, input_features: Tensor) -> Tensor: + inputs_embeds = gelu(self.conv1(input_features)) + inputs_embeds = gelu(self.conv2(inputs_embeds)) + + inputs_embeds = permute_dims(inputs_embeds, [0, 2, 1]) + embed_pos = self.embed_positions.weight + + hidden_states = inputs_embeds + embed_pos + + for idx, encoder_layer in enumerate(self.layers): + hidden_states = encoder_layer(hidden_states) + + hidden_states = self.layer_norm(hidden_states) + + return hidden_states + + +class WhisperDecoder(Module): + def __init__(self, config: WhisperConfig): + super().__init__() + + self.max_target_positions = config.max_target_positions + self.max_source_positions = config.max_source_positions + self.embed_scale = 1.0 + self.embed_tokens = Embedding(config.vocab_size, config.d_model, dtype=config.dtype) + self.embed_positions = WhisperPositionalEmbedding( + self.max_target_positions, config.d_model, dtype=config.dtype + ) + + self.layers = ModuleList([DecoderLayer(config) for _ in range(config.decoder_layers)]) + + self.layer_norm = LayerNorm(config.d_model, dtype=config.dtype) + + def forward( + self, + input_ids: Tensor, + total_seq_len: Optional[tir.Var] = None, + encoder_hidden_states: Optional[Tensor] = None, + cached_encoder_key_value: Optional[Tuple[Tuple[Tensor]]] = None, + attention_mask: Optional[Tensor] = None, + ): + # total_seq_len = Length of generated tokens + input_embeds = self.embed_tokens(input_ids) + past_seq_len = total_seq_len - 1 + position_embeds = self.embed_positions(input_ids, offset=past_seq_len) + + hidden_states = input_embeds + position_embeds + + all_encoder_key_value = () + for idx, decoder_layer in enumerate(self.layers): + ith_cached_encoder_key_value = ( + cached_encoder_key_value[idx] if cached_encoder_key_value is not None else None + ) + hidden_states, encoder_key_value = decoder_layer( + hidden_states=hidden_states, + total_seq_len=total_seq_len, + encoder_hidden_states=encoder_hidden_states, + cached_encoder_hidden_states=ith_cached_encoder_key_value, + attention_mask=attention_mask, + ) + if cached_encoder_key_value is None: + all_encoder_key_value += (encoder_key_value,) + + hidden_states = self.layer_norm(hidden_states) + + if cached_encoder_key_value is None: + return hidden_states, all_encoder_key_value + else: + return hidden_states, None + + +class WhisperModel(Module): + def __init__(self, config: WhisperConfig): + self.encoder = WhisperEncoder(config) + self.decoder = WhisperDecoder(config) + + +class WhisperForConditionalGeneration(Module): + def __init__(self, config: WhisperConfig): + self.model = WhisperModel(config) + self.proj_out = Linear(config.d_model, config.vocab_size, bias=False, dtype=config.dtype) + + def encode(self, input_features: Tensor) -> Tensor: + return self.model.encoder(input_features) + + def decode( + self, input_ids: Tensor, total_seq_len: int, encoder_hidden_states: Tensor + ) -> Tuple[Tensor, Tuple[Tuple[Tensor]]]: + hidden_states, all_encoder_key_value = self.model.decoder.forward( + input_ids=input_ids, + total_seq_len=total_seq_len, + encoder_hidden_states=encoder_hidden_states, + cached_encoder_key_value=None, + attention_mask=None, + ) + lm_logits = self.proj_out(hidden_states) + return lm_logits, all_encoder_key_value + + def prefill( + self, input_ids: Tensor, total_seq_len: int, cached_encoder_key_value: Tuple[Tuple[Tensor]] + ) -> Tensor: + hidden_states, _ = self.model.decoder.forward( + input_ids=input_ids, + total_seq_len=total_seq_len, + encoder_hidden_states=None, + cached_encoder_key_value=cached_encoder_key_value, + attention_mask=None, + ) + lm_logits = self.proj_out(hidden_states) + return lm_logits diff --git a/tests/python/test_model_whisper.py b/tests/python/test_model_whisper.py new file mode 100644 index 0000000000..40515954a9 --- /dev/null +++ b/tests/python/test_model_whisper.py @@ -0,0 +1,168 @@ +# pylint: disable=invalid-name,missing-docstring +from typing import Tuple, Any + +import torch +from transformers import WhisperForConditionalGeneration as hf_Whisper +from transformers import WhisperProcessor +from datasets import load_dataset +import torch + +import tvm +from tvm.relax.frontend.nn import spec + +from mlc_llm.models.whisper import WhisperConfig, WhisperForConditionalGeneration + + +def load_whisper_from_hf() -> Tuple[hf_Whisper, WhisperProcessor]: + processor = WhisperProcessor.from_pretrained("openai/whisper-medium") + hf_model = hf_Whisper.from_pretrained("openai/whisper-medium") + hf_model = hf_model.eval().to("cuda") + return hf_model, processor + + +def load_data(processor: WhisperProcessor, test_idx: int) -> Tuple[torch.Tensor, str]: + ds = load_dataset("hf-internal-testing/librispeech_asr_dummy", "clean", split="validation") + sample = ds[test_idx]["audio"] + text = ds[test_idx]["text"] + input_features = processor( + sample["array"], sampling_rate=sample["sampling_rate"], return_tensors="pt" + ).input_features.to("cuda") + + return input_features, text + + +def pipe(model: Any, config: WhisperConfig, input_features) -> torch.Tensor: + # encode + encode_output = model["encode"](input_features) + + # decode start token + input_ids = torch.tensor([[config.decoder_start_token_id]], dtype=torch.int32).to("cuda") + generated_tokens = [config.decoder_start_token_id] + + while True: + if len(generated_tokens) == 1: + outputs, encode_kv_cache = model["decode"]( + input_ids, len(generated_tokens), encode_output + ) + else: + outputs = model["prefill"](input_ids, len(generated_tokens), encode_kv_cache) + + outputs_logits = outputs + next_token_logits = outputs_logits[:, 0, :] + + # suppress tokens + next_tokens_scores = next_token_logits + next_tokens_scores[:, config.suppress_tokens] = -float("inf") + + # suppress tokens at begin + if len(generated_tokens) == 1 + config.forced_decoder_ids[-1][0]: + next_tokens_scores[:, config.begin_suppress_tokens] = -float("inf") + + # force tokens at sepcific position + generation_idx = len(generated_tokens) + current_token = dict(config.forced_decoder_ids).get(generation_idx, None) + if current_token is not None: + next_tokens_scores[:, :] = -float("inf") + next_tokens_scores[:, current_token] = 0 + + # argmax + next_token = torch.argmax(next_tokens_scores, dim=-1)[0] + input_ids[0][0] = next_token + + generated_tokens.append(next_token) + + # stop when we meet eos_token_id or exceed the maximum length + if ( + next_token == config.eos_token_id + or len(generated_tokens) == config.max_target_positions + ): + break + + return generated_tokens + + +def main(): + # Set the device and target + dev = tvm.cuda() + target = tvm.target.Target( + { + "kind": "cuda", + "max_shared_memory_per_block": dev.max_shared_memory_per_block, + "max_threads_per_block": dev.max_threads_per_block, + "thread_warp_size": dev.warp_size, + "registers_per_block": 65536, + "arch": "sm_" + tvm.cuda().compute_version.replace(".", ""), + } + ) + + # load model from transformers + hf_model, processor = load_whisper_from_hf() + + # Define the model config + config = WhisperConfig(**hf_model.config.to_dict()) + bsz, encode_input_ndim = 1, 16000 * 30 // 160 + + # Define the model + model = WhisperForConditionalGeneration(config=config) + + mod_spec = { + "encode": { + "input_features": spec.Tensor([bsz, config.num_mel_bins, encode_input_ndim], "float32"), + }, + "decode": { + "input_ids": spec.Tensor([bsz, "seq_len"], "int32"), + "total_seq_len": int, + "encoder_hidden_states": spec.Tensor( + [bsz, config.max_source_positions, config.d_model], "float32" + ), + }, + "prefill": { + "input_ids": spec.Tensor([bsz, 1], "int32"), + "total_seq_len": int, + "cached_encoder_key_value": tuple( + tuple( + spec.Tensor( + [ + 1, + config.max_source_positions, + config.decoder_attention_heads, + config.d_model // config.decoder_attention_heads, + ], + "float32", + ) + for i2 in range(2) + ) + for i1 in range(config.num_hidden_layers) + ), + }, + } + + # Usercase1, export it to TVM's IRModule, use `mod.show()` to print the IRModule + mod, _ = model.export_tvm(spec=mod_spec) + + # Usercase2, JIT compile a model + for name, param in model.state_dict().items(): + param.data = hf_model.state_dict()[name] + + model = model.jit(spec=mod_spec, target=target, device="cuda", out_format="torch", debug=True) + + # Test on librispeech_asr_dummy + input_features, text = load_data(processor, test_idx=0) + generated_tokens = pipe(model, config, input_features) + + # compare with hf whisper output + hf_predicted_ids = hf_model.generate(input_features).to("cpu") + assert torch.equal(torch.tensor([generated_tokens], dtype=torch.long), hf_predicted_ids) + + # decode token ids to text + output = processor.decode(generated_tokens, skip_special_tokens=True) + assert ( + output + == " Mr. Quilter is the apostle of the middle classes and we are glad to welcome his gospel." + ) + + print("Transcription:\n", output) + + +if __name__ == "__main__": + main() From 860632763ac25e06a59cce18c251e464cafb8673 Mon Sep 17 00:00:00 2001 From: Lesheng Jin Date: Mon, 5 Feb 2024 20:57:50 +0000 Subject: [PATCH 2/3] fix --- mlc_llm/models/whisper.py | 430 -------------------------------------- 1 file changed, 430 deletions(-) delete mode 100644 mlc_llm/models/whisper.py diff --git a/mlc_llm/models/whisper.py b/mlc_llm/models/whisper.py deleted file mode 100644 index 686a996cba..0000000000 --- a/mlc_llm/models/whisper.py +++ /dev/null @@ -1,430 +0,0 @@ -import dataclasses -from typing import Tuple, Optional - -import tvm -from tvm import te, tir -import tvm.relax.frontend.nn as nn -from tvm.relax.frontend.nn import ( - Embedding, - KVCache, - Linear, - LayerNorm, - Conv1D, - Module, - ModuleList, - Parameter, - Tensor, - tensor_expr_op, - permute_dims, - reshape, - squeeze, - matmul, - maximum, - minimum, - softmax, - gelu, - print_, -) - - -@dataclasses.dataclass -class WhisperConfig: - def __init__( - self, - dtype="float32", - vocab_size=51865, - num_mel_bins=80, - encoder_layers=6, - encoder_attention_heads=4, - decoder_layers=6, - decoder_attention_heads=4, - decoder_ffn_dim=1536, - encoder_ffn_dim=1536, - decoder_start_token_id=50257, - d_model=256, - max_source_positions=1500, - max_target_positions=448, - pad_token_id=50256, - bos_token_id=50257, - eos_token_id=50256, - suppress_tokens=None, - begin_suppress_tokens=[220, 50256], - forced_decoder_ids=None, - **kwargs, - ): - self.dtype = dtype - self.vocab_size = vocab_size - self.num_mel_bins = num_mel_bins - self.d_model = d_model - self.encoder_layers = encoder_layers - self.encoder_attention_heads = encoder_attention_heads - self.decoder_layers = decoder_layers - self.decoder_attention_heads = decoder_attention_heads - self.decoder_ffn_dim = decoder_ffn_dim - self.encoder_ffn_dim = encoder_ffn_dim - self.num_hidden_layers = encoder_layers - self.max_source_positions = max_source_positions - self.max_target_positions = max_target_positions - self.pad_token_id = pad_token_id - self.bos_token_id = bos_token_id - self.eos_token_id = eos_token_id - self.decoder_start_token_id = decoder_start_token_id - self.suppress_tokens = suppress_tokens - self.begin_suppress_tokens = begin_suppress_tokens - self.forced_decoder_ids = forced_decoder_ids - self.kwargs = kwargs - - -class WhisperPositionalEmbedding(Module): - def __init__(self, max_seq_len: int, embed_dim: int, dtype: str): - super().__init__() - self.max_seq_len = max_seq_len - self.embed_dim = embed_dim - self.weight = Parameter((max_seq_len, embed_dim), dtype=dtype) - - def forward(self, x: Tensor, offset: tir.Var): - def te_op(x: te.Tensor, embed: te.Tensor, offset: tir.Var): - def compute(i: tir.Var, j: tir.Var, k: tir.Var): - return embed[offset + j, k] - - return tvm.te.compute([*x.shape, embed.shape[-1]], compute, name="position_embedding") - - pos_embed = tensor_expr_op(te_op, "position_embedding", args=[x, self.weight, offset]) - return pos_embed - - -class WhisperAttention(Module): - def __init__( - self, embed_dim: int, num_heads: int, kv_cache_len: int, dtype: str, bias: bool = True - ): - super().__init__() - self.embed_dim = embed_dim - self.num_heads = num_heads - self.head_dim = self.embed_dim // self.num_heads - - if (self.head_dim * num_heads) != self.embed_dim: - raise ValueError( - f"embed_dim must be divisible by num_heads (got `embed_dim`: {self.embed_dim}" - f" and `num_heads`: {num_heads})." - ) - self.scaling = self.head_dim**-0.5 - self.k_proj = Linear(self.embed_dim, self.embed_dim, dtype=dtype, bias=False) - self.v_proj = Linear(self.embed_dim, self.embed_dim, dtype=dtype, bias=bias) - self.q_proj = Linear(self.embed_dim, self.embed_dim, dtype=dtype, bias=bias) - self.out_proj = Linear(self.embed_dim, self.embed_dim, dtype=dtype, bias=bias) - if kv_cache_len > 0: - self.k_cache = KVCache(kv_cache_len, [self.num_heads, self.head_dim]) - self.v_cache = KVCache(kv_cache_len, [self.num_heads, self.head_dim]) - - def forward( - self, - hidden_states: Tensor, - key_value_states: Optional[Tensor] = None, - cached_cross_attn_states: Optional[Tuple[Tensor]] = None, - attention_mask: Optional[Tensor] = None, - total_seq_len: Optional[tir.Var] = None, - ) -> Tuple[Tensor, Optional[Tuple[Tensor]]]: - is_cross_attention = key_value_states is not None or cached_cross_attn_states is not None - - h, d = self.num_heads, self.head_dim - bsz, q_len, _ = hidden_states.shape - assert bsz == 1, "Only support batch size 1 at this moment." - - q = reshape(self.q_proj(hidden_states) * self.scaling, (bsz, q_len, h, d)) - - if is_cross_attention: - # cross attention - if cached_cross_attn_states is None: - # no cache, cross attentions - kv_len = key_value_states.shape[1] - k = reshape(self.k_proj(key_value_states), (bsz, kv_len, h, d)) - v = reshape(self.v_proj(key_value_states), (bsz, kv_len, h, d)) - cached_kv = (k, v) - else: - # reuse cached k,v, cross_attentions - k, v = cached_cross_attn_states - else: - # self attention - k = reshape(self.k_proj(hidden_states), (bsz, q_len, h, d)) - v = reshape(self.v_proj(hidden_states), (bsz, q_len, h, d)) - - if total_seq_len is not None: - # reuse cached k, v, self_attention - self.k_cache.append(squeeze(k, axis=0)) - self.v_cache.append(squeeze(v, axis=0)) - k = reshape(self.k_cache.view(total_seq_len), (bsz, total_seq_len, h, d)) - v = reshape(self.v_cache.view(total_seq_len), (bsz, total_seq_len, h, d)) - else: - # encode self attention, no cache - ... - - q = permute_dims(q, [0, 2, 1, 3]) # [b, h, q_len, d] - k = permute_dims(k, [0, 2, 1, 3]) # [b, h, q_len, d] - v = permute_dims(v, [0, 2, 1, 3]) # [b, h, q_len, d] - - attn_weights = matmul(q, (permute_dims(k, [0, 1, 3, 2]))) # [b, h, q_len, q_len] - - if attention_mask is not None: - attn_weights = attn_weights + attention_mask - - dtype = attn_weights.dtype - attn_weights = attn_weights.maximum(tir.min_value(dtype)) - attn_weights = attn_weights.minimum(tir.max_value(dtype)) - if dtype == "float32": - attn_weights = softmax(attn_weights, axis=-1) - else: - attn_weights = softmax(attn_weights.astype("float32"), axis=-1).astype(dtype) - attn_output = matmul(attn_weights, v) # [b, h, q_len, d] - - attn_output = permute_dims(attn_output, [0, 2, 1, 3]) # [b, q_len, h, d] - attn_output = reshape(attn_output, (bsz, q_len, self.embed_dim)) # [b, q_len, h * d] - - attn_output = self.out_proj(attn_output) - - if is_cross_attention and cached_cross_attn_states is None: - return attn_output, cached_kv - else: - return attn_output, None - - -class EncoderLayer(Module): - def __init__(self, config: WhisperConfig): - super().__init__() - self.embed_dim = config.d_model - self.self_attn = WhisperAttention( - embed_dim=self.embed_dim, - num_heads=config.encoder_attention_heads, - kv_cache_len=0, # no need for kv_cache - dtype=config.dtype, - ) - self.self_attn_layer_norm = LayerNorm(self.embed_dim, dtype=config.dtype) - self.fc1 = Linear(self.embed_dim, config.encoder_ffn_dim, dtype=config.dtype) - self.fc2 = Linear(config.encoder_ffn_dim, self.embed_dim, dtype=config.dtype) - self.final_layer_norm = LayerNorm(self.embed_dim, dtype=config.dtype) - - def forward(self, hidden_states: Tensor, attention_mask: Optional[Tensor] = None) -> Tensor: - residual = hidden_states - hidden_states = self.self_attn_layer_norm(hidden_states) - hidden_states, _ = self.self_attn( - hidden_states=hidden_states, - key_value_states=None, - cached_cross_attn_states=None, - attention_mask=attention_mask, - total_seq_len=None, - ) - hidden_states = residual + hidden_states - - residual = hidden_states - hidden_states = self.final_layer_norm(hidden_states) - hidden_states = gelu(self.fc1(hidden_states)) - hidden_states = self.fc2(hidden_states) - hidden_states = residual + hidden_states - hidden_states = hidden_states.maximum(tir.min_value(hidden_states.dtype)) - hidden_states = hidden_states.minimum(tir.max_value(hidden_states.dtype)) - - return hidden_states - - -class DecoderLayer(Module): - def __init__(self, config: WhisperConfig): - super().__init__() - self.embed_dim = config.d_model - - self.self_attn = WhisperAttention( - embed_dim=self.embed_dim, - num_heads=config.decoder_attention_heads, - kv_cache_len=100, # TODO - dtype=config.dtype, - ) - - self.self_attn_layer_norm = LayerNorm(self.embed_dim, dtype=config.dtype) - self.encoder_attn = WhisperAttention( - self.embed_dim, - config.decoder_attention_heads, - kv_cache_len=100, # TODO - dtype=config.dtype, - ) - self.encoder_attn_layer_norm = LayerNorm(self.embed_dim, dtype=config.dtype) - self.fc1 = Linear(self.embed_dim, config.decoder_ffn_dim, dtype=config.dtype) - self.fc2 = Linear(config.decoder_ffn_dim, self.embed_dim, dtype=config.dtype) - self.final_layer_norm = LayerNorm(self.embed_dim, dtype=config.dtype) - - def forward( - self, - hidden_states: Tensor, - encoder_hidden_states: Tensor, - cached_encoder_hidden_states: Tensor, - total_seq_len: tir.Var, - attention_mask: Optional[Tensor] = None, - encoder_attention_mask: Optional[Tensor] = None, - ) -> Tuple[Tensor, Optional[Tuple[Tensor]]]: - residual = hidden_states - hidden_states = self.self_attn_layer_norm(hidden_states) - - # Self Attention - hidden_states, _ = self.self_attn( - hidden_states=hidden_states, - total_seq_len=total_seq_len, - key_value_states=None, - cached_cross_attn_states=None, - attention_mask=attention_mask, - ) - hidden_states = residual + hidden_states - - # Cross-Attention Block - residual = hidden_states - hidden_states = self.encoder_attn_layer_norm(hidden_states) - hidden_states, cross_attn_key_value = self.encoder_attn( - hidden_states=hidden_states, - total_seq_len=total_seq_len, - key_value_states=encoder_hidden_states, - cached_cross_attn_states=cached_encoder_hidden_states, - attention_mask=encoder_attention_mask, - ) - hidden_states = residual + hidden_states - - # Fully Connected - residual = hidden_states - hidden_states = self.final_layer_norm(hidden_states) - hidden_states = gelu(self.fc1(hidden_states)) - hidden_states = self.fc2(hidden_states) - hidden_states = residual + hidden_states - - if cached_encoder_hidden_states is None: - return hidden_states, cross_attn_key_value - else: - return hidden_states, None - - -class WhisperEncoder(Module): - def __init__(self, config: WhisperConfig): - super().__init__() - - embed_dim = config.d_model - self.num_mel_bins = config.num_mel_bins - self.padding_idx = config.pad_token_id - self.max_source_positions = config.max_source_positions - self.embed_scale = 1.0 - - self.conv1 = Conv1D( - self.num_mel_bins, embed_dim, kernel_size=3, padding=1, dtype=config.dtype - ) - self.conv2 = Conv1D( - embed_dim, embed_dim, kernel_size=3, stride=2, padding=1, dtype=config.dtype - ) - - self.embed_positions = Embedding(self.max_source_positions, embed_dim, dtype=config.dtype) - - self.layers = ModuleList([EncoderLayer(config) for _ in range(config.encoder_layers)]) - self.layer_norm = LayerNorm(config.d_model, dtype=config.dtype) - - def forward(self, input_features: Tensor) -> Tensor: - inputs_embeds = gelu(self.conv1(input_features)) - inputs_embeds = gelu(self.conv2(inputs_embeds)) - - inputs_embeds = permute_dims(inputs_embeds, [0, 2, 1]) - embed_pos = self.embed_positions.weight - - hidden_states = inputs_embeds + embed_pos - - for idx, encoder_layer in enumerate(self.layers): - hidden_states = encoder_layer(hidden_states) - - hidden_states = self.layer_norm(hidden_states) - - return hidden_states - - -class WhisperDecoder(Module): - def __init__(self, config: WhisperConfig): - super().__init__() - - self.max_target_positions = config.max_target_positions - self.max_source_positions = config.max_source_positions - self.embed_scale = 1.0 - self.embed_tokens = Embedding(config.vocab_size, config.d_model, dtype=config.dtype) - self.embed_positions = WhisperPositionalEmbedding( - self.max_target_positions, config.d_model, dtype=config.dtype - ) - - self.layers = ModuleList([DecoderLayer(config) for _ in range(config.decoder_layers)]) - - self.layer_norm = LayerNorm(config.d_model, dtype=config.dtype) - - def forward( - self, - input_ids: Tensor, - total_seq_len: Optional[tir.Var] = None, - encoder_hidden_states: Optional[Tensor] = None, - cached_encoder_key_value: Optional[Tuple[Tuple[Tensor]]] = None, - attention_mask: Optional[Tensor] = None, - ): - # total_seq_len = Length of generated tokens - input_embeds = self.embed_tokens(input_ids) - past_seq_len = total_seq_len - 1 - position_embeds = self.embed_positions(input_ids, offset=past_seq_len) - - hidden_states = input_embeds + position_embeds - - all_encoder_key_value = () - for idx, decoder_layer in enumerate(self.layers): - ith_cached_encoder_key_value = ( - cached_encoder_key_value[idx] if cached_encoder_key_value is not None else None - ) - hidden_states, encoder_key_value = decoder_layer( - hidden_states=hidden_states, - total_seq_len=total_seq_len, - encoder_hidden_states=encoder_hidden_states, - cached_encoder_hidden_states=ith_cached_encoder_key_value, - attention_mask=attention_mask, - ) - if cached_encoder_key_value is None: - all_encoder_key_value += (encoder_key_value,) - - hidden_states = self.layer_norm(hidden_states) - - if cached_encoder_key_value is None: - return hidden_states, all_encoder_key_value - else: - return hidden_states, None - - -class WhisperModel(Module): - def __init__(self, config: WhisperConfig): - self.encoder = WhisperEncoder(config) - self.decoder = WhisperDecoder(config) - - -class WhisperForConditionalGeneration(Module): - def __init__(self, config: WhisperConfig): - self.model = WhisperModel(config) - self.proj_out = Linear(config.d_model, config.vocab_size, bias=False, dtype=config.dtype) - - def encode(self, input_features: Tensor) -> Tensor: - return self.model.encoder(input_features) - - def decode( - self, input_ids: Tensor, total_seq_len: int, encoder_hidden_states: Tensor - ) -> Tuple[Tensor, Tuple[Tuple[Tensor]]]: - hidden_states, all_encoder_key_value = self.model.decoder.forward( - input_ids=input_ids, - total_seq_len=total_seq_len, - encoder_hidden_states=encoder_hidden_states, - cached_encoder_key_value=None, - attention_mask=None, - ) - lm_logits = self.proj_out(hidden_states) - return lm_logits, all_encoder_key_value - - def prefill( - self, input_ids: Tensor, total_seq_len: int, cached_encoder_key_value: Tuple[Tuple[Tensor]] - ) -> Tensor: - hidden_states, _ = self.model.decoder.forward( - input_ids=input_ids, - total_seq_len=total_seq_len, - encoder_hidden_states=None, - cached_encoder_key_value=cached_encoder_key_value, - attention_mask=None, - ) - lm_logits = self.proj_out(hidden_states) - return lm_logits From c88231ba031ceeb44b95bd264d83b1e6202819c3 Mon Sep 17 00:00:00 2001 From: Lesheng Jin Date: Fri, 1 Mar 2024 07:10:28 +0000 Subject: [PATCH 3/3] whisper checkpoint --- python/mlc_chat/__init__.py | 3 +- python/mlc_chat/base.py | 5 +- python/mlc_chat/compiler_pass/pipeline.py | 1 + python/mlc_chat/interface/compile.py | 1 + python/mlc_chat/model/model.py | 14 + python/mlc_chat/model/whisper/__init__.py | 0 .../mlc_chat/model/whisper/whisper_loader.py | 51 + .../mlc_chat/model/whisper/whisper_model.py | 451 ++ .../model/whisper/whisper_quantization.py | 40 + .../quantization/group_quantization.py | 41 +- test_cli.py | 32 + test_disco.py | 37 + test_disco_batched.py | 85 + test_gpt.sh | 22 + test_llama.sh | 20 + test_mistral.sh | 22 + test_multi_gpu.sh | 14 + test_old.sh | 5 + test_phi.sh | 30 + test_red_pajama.sh | 20 + test_slm.py | 91 + test_slm_disco.py | 132 + test_whisper.py | 205 + test_whisper.sh | 21 + test_yi.sh | 19 + tests/python/test_model_whisper.py | 13 +- wasted/1.py | 4652 +++++++++++++++++ wasted/codetest1.py | 24 + wasted/deploy_mlc.py | 102 + wasted/deploy_new_mlc.py | 110 + wasted/disco_test/0.npy | Bin 0 -> 35968 bytes wasted/disco_test/1.npy | Bin 0 -> 35968 bytes wasted/disco_test/2.npy | Bin 0 -> 35968 bytes wasted/disco_test/3.npy | Bin 0 -> 35968 bytes wasted/disco_test/single.npy | Bin 0 -> 35968 bytes wasted/gtable_to_json.py | 18 + wasted/load_np.py | 56 + wasted/model_info.json | 1 + wasted/phi_model_old.py | 451 ++ wasted/sample_mlc_chat.py | 27 + wasted/sheet.csv | 29 + wasted/test000.py | 54 + wasted/test_7mistral.sh | 11 + wasted/test_awq.sh | 10 + wasted/test_awq_gemm.sh | 14 + wasted/test_decode_awq_gemm.py | 110 + wasted/test_mistral.py | 54 + wasted/test_mistral_multi_gpu.sh | 19 + wasted/test_mlc.py | 30 + wasted/test_mlc_hf.py | 23 + wasted/test_mlc_hf_batch.py | 81 + wasted/test_mlc_phi.py | 16 + wasted/test_mlc_red_pajama.py | 14 + wasted/test_old_red_pajama.sh | 3 + wasted/test_phi_hf.py | 19 + wasted/test_phi_mlc.py | 67 + wasted/test_preshard.sh | 7 + wasted/worker.py | 64 + 58 files changed, 7416 insertions(+), 25 deletions(-) create mode 100644 python/mlc_chat/model/whisper/__init__.py create mode 100644 python/mlc_chat/model/whisper/whisper_loader.py create mode 100644 python/mlc_chat/model/whisper/whisper_model.py create mode 100644 python/mlc_chat/model/whisper/whisper_quantization.py create mode 100644 test_cli.py create mode 100644 test_disco.py create mode 100644 test_disco_batched.py create mode 100755 test_gpt.sh create mode 100755 test_llama.sh create mode 100755 test_mistral.sh create mode 100755 test_multi_gpu.sh create mode 100755 test_old.sh create mode 100755 test_phi.sh create mode 100755 test_red_pajama.sh create mode 100644 test_slm.py create mode 100644 test_slm_disco.py create mode 100644 test_whisper.py create mode 100755 test_whisper.sh create mode 100755 test_yi.sh create mode 100644 wasted/1.py create mode 100644 wasted/codetest1.py create mode 100644 wasted/deploy_mlc.py create mode 100644 wasted/deploy_new_mlc.py create mode 100644 wasted/disco_test/0.npy create mode 100644 wasted/disco_test/1.npy create mode 100644 wasted/disco_test/2.npy create mode 100644 wasted/disco_test/3.npy create mode 100644 wasted/disco_test/single.npy create mode 100644 wasted/gtable_to_json.py create mode 100644 wasted/load_np.py create mode 100644 wasted/model_info.json create mode 100644 wasted/phi_model_old.py create mode 100644 wasted/sample_mlc_chat.py create mode 100644 wasted/sheet.csv create mode 100644 wasted/test000.py create mode 100755 wasted/test_7mistral.sh create mode 100755 wasted/test_awq.sh create mode 100755 wasted/test_awq_gemm.sh create mode 100644 wasted/test_decode_awq_gemm.py create mode 100644 wasted/test_mistral.py create mode 100644 wasted/test_mistral_multi_gpu.sh create mode 100644 wasted/test_mlc.py create mode 100644 wasted/test_mlc_hf.py create mode 100644 wasted/test_mlc_hf_batch.py create mode 100644 wasted/test_mlc_phi.py create mode 100644 wasted/test_mlc_red_pajama.py create mode 100755 wasted/test_old_red_pajama.sh create mode 100644 wasted/test_phi_hf.py create mode 100644 wasted/test_phi_mlc.py create mode 100755 wasted/test_preshard.sh create mode 100644 wasted/worker.py diff --git a/python/mlc_chat/__init__.py b/python/mlc_chat/__init__.py index f577e0308e..351274a2f3 100644 --- a/python/mlc_chat/__init__.py +++ b/python/mlc_chat/__init__.py @@ -2,6 +2,7 @@ MLC Chat is the app runtime of MLC LLM. """ -from . import protocol, serve + +# from . import protocol, serve from .chat_module import ChatConfig, ChatModule, ConvConfig, GenerationConfig from .libinfo import __version__ diff --git a/python/mlc_chat/base.py b/python/mlc_chat/base.py index 13c7ba9f84..b5db400d79 100644 --- a/python/mlc_chat/base.py +++ b/python/mlc_chat/base.py @@ -1,4 +1,5 @@ """Load MLC LLM library and _ffi_api functions.""" + import ctypes import os import sys @@ -24,5 +25,5 @@ def _load_mlc_llm_lib(): # only load once here -if SKIP_LOADING_MLCLLM_SO == "0": - _LIB, _LIB_PATH = _load_mlc_llm_lib() +# if SKIP_LOADING_MLCLLM_SO == "0": +# _LIB, _LIB_PATH = _load_mlc_llm_lib() diff --git a/python/mlc_chat/compiler_pass/pipeline.py b/python/mlc_chat/compiler_pass/pipeline.py index f7d0e0e8ab..fc8971bc55 100644 --- a/python/mlc_chat/compiler_pass/pipeline.py +++ b/python/mlc_chat/compiler_pass/pipeline.py @@ -1,4 +1,5 @@ """The compilation pipeline for LLM applications.""" + from pathlib import Path from typing import Any, Dict, List, Optional diff --git a/python/mlc_chat/interface/compile.py b/python/mlc_chat/interface/compile.py index 7f7d8a28d3..8a043f97e2 100644 --- a/python/mlc_chat/interface/compile.py +++ b/python/mlc_chat/interface/compile.py @@ -1,4 +1,5 @@ """Python entrypoint of compilation.""" + import dataclasses import math from io import StringIO diff --git a/python/mlc_chat/model/model.py b/python/mlc_chat/model/model.py index a7fefdeca3..e6c3e4b41c 100644 --- a/python/mlc_chat/model/model.py +++ b/python/mlc_chat/model/model.py @@ -17,6 +17,7 @@ from .phi import phi_loader, phi_model, phi_quantization from .qwen import qwen_loader, qwen_model, qwen_quantization from .stable_lm import stablelm_loader, stablelm_model, stablelm_quantization +from .whisper import whisper_loader, whisper_model, whisper_quantization ModelConfig = Any """A ModelConfig is an object that represents a model architecture. It is required to have @@ -195,4 +196,17 @@ class Model: "group-quant": stablelm_quantization.group_quant, }, ), + "whisper": Model( + name="whisper", + model=whisper_model.WhisperForConditionalGeneration, + config=whisper_model.WhisperConfig, + source={ + "huggingface-torch": whisper_loader.huggingface, + "huggingface-safetensor": whisper_loader.huggingface, + }, + quantize={ + "no-quant": whisper_quantization.no_quant, + "group-quant": whisper_quantization.group_quant, + }, + ), } diff --git a/python/mlc_chat/model/whisper/__init__.py b/python/mlc_chat/model/whisper/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/python/mlc_chat/model/whisper/whisper_loader.py b/python/mlc_chat/model/whisper/whisper_loader.py new file mode 100644 index 0000000000..a5394d39ed --- /dev/null +++ b/python/mlc_chat/model/whisper/whisper_loader.py @@ -0,0 +1,51 @@ +""" +This file specifies how MLC's Whisper parameter maps from other formats, for example HuggingFace +PyTorch, HuggingFace safetensors. +""" + +import functools + +from mlc_chat.loader import ExternMapping +from mlc_chat.quantization import Quantization + +from .whisper_model import WhisperConfig, WhisperForConditionalGeneration + + +def huggingface(model_config: WhisperConfig, quantization: Quantization) -> ExternMapping: + """Returns a parameter mapping that maps from the names of MLC LLM parameters to + the names of HuggingFace PyTorch parameters. + + Parameters + ---------- + model_config : WhisperConfig + The configuration of the GPTNeoX model. + + quantization : Quantization + The quantization configuration. + + Returns + ------- + param_map : ExternMapping + The parameter mapping from MLC to HuggingFace PyTorch. + """ + model = WhisperForConditionalGeneration(model_config) + if quantization is not None: + model.to(quantization.model_dtype) + _, _named_params, _ = model.export_tvm( # type: ignore[misc] + spec=model.get_default_spec(), + allow_extern=True, + ) + named_parameters = dict(_named_params) + + mapping = ExternMapping() + + for mlc_name, mlc_param in named_parameters.items(): + mapping.add_mapping( + mlc_name, + [mlc_name], + functools.partial( + lambda x, dtype: x.astype(dtype), + dtype=mlc_param.dtype, + ), + ) + return mapping diff --git a/python/mlc_chat/model/whisper/whisper_model.py b/python/mlc_chat/model/whisper/whisper_model.py new file mode 100644 index 0000000000..431ee7fce7 --- /dev/null +++ b/python/mlc_chat/model/whisper/whisper_model.py @@ -0,0 +1,451 @@ +""" +Implementation for Whisper architecture. +""" + +import dataclasses +from typing import Any, Dict, List, Optional, Tuple + +import tvm +from tvm import te, tir +from tvm.relax.frontend import nn + +from mlc_chat import op as op_ext +from mlc_chat.support import logging +from mlc_chat.support.config import ConfigBase + +logger = logging.getLogger(__name__) + + +@dataclasses.dataclass +class WhisperConfig(ConfigBase): + """Configuration of the Whipser model.""" + + vocab_size: int + num_hidden_layers: int + num_mel_bins: int + encoder_layers: int + encoder_attention_heads: int + decoder_layers: int + decoder_attention_heads: int + decoder_ffn_dim: int + encoder_ffn_dim: int + decoder_start_token_id: int + d_model: int + max_source_positions: int + max_target_positions: int + pad_token_id: int + bos_token_id: int + eos_token_id: int + begin_suppress_tokens: List[int] + suppress_tokens: List[int] = dataclasses.field(default_factory=list) + forced_decoder_ids: List[int] = dataclasses.field(default_factory=list) + tensor_parallel_shards: int = 1 + kwargs: Dict[str, Any] = dataclasses.field(default_factory=dict) + + def __post_init__(self): + self.context_window_size = self.max_source_positions + self.prefill_chunk_size = self.max_source_positions + + +# pylint: disable=invalid-name,missing-docstring + + +class WhisperPositionalEmbedding(nn.Module): + def __init__(self, max_seq_len: int, embed_dim: int): + super().__init__() + self.max_seq_len = max_seq_len + self.embed_dim = embed_dim + self.weight = nn.Parameter( + (max_seq_len, embed_dim), + ) + + def forward(self, x: nn.Tensor, offset: tir.Var): + def te_op(x: te.Tensor, embed: te.Tensor, offset: tir.Var): + def compute(_: tir.Var, j: tir.Var, k: tir.Var): + return embed[offset + j, k] + + return tvm.te.compute([*x.shape, embed.shape[-1]], compute, name="position_embedding") + + pos_embed = nn.tensor_expr_op(te_op, "position_embedding", args=[x, self.weight, offset]) + return pos_embed + + +class WhisperAttention(nn.Module): + def __init__(self, embed_dim: int, num_heads: int, kv_cache_len: int, bias: bool = True): + super().__init__() + self.embed_dim = embed_dim + self.num_heads = num_heads + self.head_dim = self.embed_dim // self.num_heads + + if (self.head_dim * num_heads) != self.embed_dim: + raise ValueError( + f"embed_dim must be divisible by num_heads (got `embed_dim`: {self.embed_dim}" + f" and `num_heads`: {num_heads})." + ) + self.scaling = self.head_dim**-0.5 + self.k_proj = nn.Linear(self.embed_dim, self.embed_dim, bias=False) + self.v_proj = nn.Linear(self.embed_dim, self.embed_dim, bias=bias) + self.q_proj = nn.Linear(self.embed_dim, self.embed_dim, bias=bias) + self.out_proj = nn.Linear(self.embed_dim, self.embed_dim, bias=bias) + if kv_cache_len > 0: + self.k_cache = nn.KVCache(kv_cache_len, [self.num_heads, self.head_dim]) + self.v_cache = nn.KVCache(kv_cache_len, [self.num_heads, self.head_dim]) + + def forward( + self, + hidden_states: nn.Tensor, + key_value_states: Optional[nn.Tensor] = None, + cached_cross_attn_states: Optional[Tuple[nn.Tensor, nn.Tensor]] = None, + attention_mask: Optional[nn.Tensor] = None, + total_seq_len: Optional[tir.Var] = None, + ) -> Tuple[nn.Tensor, Optional[Tuple[nn.Tensor, nn.Tensor]]]: + is_cross_attention = key_value_states is not None or cached_cross_attn_states is not None + + h, d = self.num_heads, self.head_dim + bsz, q_len, _ = hidden_states.shape + assert bsz == 1, "Only support batch size 1 at this moment." + + q = nn.reshape(self.q_proj(hidden_states) * self.scaling, (bsz, q_len, h, d)) + + if is_cross_attention: + # cross attention + if cached_cross_attn_states is None: + # no cache, cross attentions + kv_len = key_value_states.shape[1] + k = nn.reshape(self.k_proj(key_value_states), (bsz, kv_len, h, d)) + v = nn.reshape(self.v_proj(key_value_states), (bsz, kv_len, h, d)) + cached_kv = (k, v) + else: + # reuse cached k,v, cross_attentions + k, v = cached_cross_attn_states + else: + # self attention + k = nn.reshape(self.k_proj(hidden_states), (bsz, q_len, h, d)) + v = nn.reshape(self.v_proj(hidden_states), (bsz, q_len, h, d)) + + if total_seq_len is not None: + # reuse cached k, v, self_attention + self.k_cache.append(nn.squeeze(k, axis=0)) + self.v_cache.append(nn.squeeze(v, axis=0)) + k = nn.reshape(self.k_cache.view(total_seq_len), (bsz, total_seq_len, h, d)) + v = nn.reshape(self.v_cache.view(total_seq_len), (bsz, total_seq_len, h, d)) + else: + # encode self attention, no cache + ... + + q = nn.permute_dims(q, [0, 2, 1, 3]) # [b, h, q_len, d] + k = nn.permute_dims(k, [0, 2, 1, 3]) # [b, h, q_len, d] + v = nn.permute_dims(v, [0, 2, 1, 3]) # [b, h, q_len, d] + + attn_weights = nn.matmul(q, (nn.permute_dims(k, [0, 1, 3, 2]))) # [b, h, q_len, q_len] + + if attention_mask is not None: + attn_weights = attn_weights + attention_mask + + dtype = attn_weights.dtype + attn_weights = attn_weights.maximum(tir.min_value(dtype)) + attn_weights = attn_weights.minimum(tir.max_value(dtype)) + if dtype == "float32": + attn_weights = nn.softmax(attn_weights, axis=-1) + else: + attn_weights = nn.softmax(attn_weights.astype("float32"), axis=-1).astype(dtype) + attn_output = nn.matmul(attn_weights, v) # [b, h, q_len, d] + + attn_output = nn.permute_dims(attn_output, [0, 2, 1, 3]) # [b, q_len, h, d] + attn_output = nn.reshape(attn_output, (bsz, q_len, self.embed_dim)) # [b, q_len, h * d] + + attn_output = self.out_proj(attn_output) + + if is_cross_attention and cached_cross_attn_states is None: + return attn_output, cached_kv + + return attn_output, None + + +class EncoderLayer(nn.Module): + def __init__(self, config: WhisperConfig): + super().__init__() + self.embed_dim = config.d_model + self.self_attn = WhisperAttention( + embed_dim=self.embed_dim, + num_heads=config.encoder_attention_heads, + kv_cache_len=0, # no need for kv_cache + ) + self.self_attn_layer_norm = nn.LayerNorm(self.embed_dim) + self.fc1 = nn.Linear(self.embed_dim, config.encoder_ffn_dim) + self.fc2 = nn.Linear(config.encoder_ffn_dim, self.embed_dim) + self.final_layer_norm = nn.LayerNorm(self.embed_dim) + + def forward( + self, hidden_states: nn.Tensor, attention_mask: Optional[nn.Tensor] = None + ) -> nn.Tensor: + residual = hidden_states + hidden_states = self.self_attn_layer_norm(hidden_states) + hidden_states, _ = self.self_attn( + hidden_states=hidden_states, + key_value_states=None, + cached_cross_attn_states=None, + attention_mask=attention_mask, + total_seq_len=None, + ) + hidden_states = residual + hidden_states + + residual = hidden_states + hidden_states = self.final_layer_norm(hidden_states) + hidden_states = nn.gelu(self.fc1(hidden_states)) + hidden_states = self.fc2(hidden_states) + hidden_states = residual + hidden_states + hidden_states = hidden_states.maximum(tir.min_value(hidden_states.dtype)) + hidden_states = hidden_states.minimum(tir.max_value(hidden_states.dtype)) + + return hidden_states + + +class DecoderLayer(nn.Module): + def __init__(self, config: WhisperConfig): + super().__init__() + self.embed_dim = config.d_model + + self.self_attn = WhisperAttention( + embed_dim=self.embed_dim, + num_heads=config.decoder_attention_heads, + kv_cache_len=config.max_target_positions, + ) + + self.self_attn_layer_norm = nn.LayerNorm(self.embed_dim) + self.encoder_attn = WhisperAttention( + self.embed_dim, + config.decoder_attention_heads, + kv_cache_len=config.max_target_positions, + ) + self.encoder_attn_layer_norm = nn.LayerNorm(self.embed_dim) + self.fc1 = nn.Linear(self.embed_dim, config.decoder_ffn_dim) + self.fc2 = nn.Linear(config.decoder_ffn_dim, self.embed_dim) + self.final_layer_norm = nn.LayerNorm(self.embed_dim) + + def forward( + self, + hidden_states: nn.Tensor, + encoder_hidden_states: nn.Tensor, + cached_encoder_hidden_states: nn.Tensor, + total_seq_len: tir.Var, + attention_mask: Optional[nn.Tensor] = None, + encoder_attention_mask: Optional[nn.Tensor] = None, + ) -> Tuple[nn.Tensor, Optional[Tuple[nn.Tensor]]]: + residual = hidden_states + hidden_states = self.self_attn_layer_norm(hidden_states) + + # Self Attention + hidden_states, _ = self.self_attn( + hidden_states=hidden_states, + total_seq_len=total_seq_len, + key_value_states=None, + cached_cross_attn_states=None, + attention_mask=attention_mask, + ) + hidden_states = residual + hidden_states + + # Cross-Attention Block + residual = hidden_states + hidden_states = self.encoder_attn_layer_norm(hidden_states) + hidden_states, cross_attn_key_value = self.encoder_attn( + hidden_states=hidden_states, + total_seq_len=total_seq_len, + key_value_states=encoder_hidden_states, + cached_cross_attn_states=cached_encoder_hidden_states, + attention_mask=encoder_attention_mask, + ) + hidden_states = residual + hidden_states + + # Fully Connected + residual = hidden_states + hidden_states = self.final_layer_norm(hidden_states) + hidden_states = nn.gelu(self.fc1(hidden_states)) + hidden_states = self.fc2(hidden_states) + hidden_states = residual + hidden_states + + if cached_encoder_hidden_states is None: + return hidden_states, cross_attn_key_value + return hidden_states, None + + +class WhisperEncoder(nn.Module): + def __init__(self, config: WhisperConfig): + super().__init__() + + embed_dim = config.d_model + self.num_mel_bins = config.num_mel_bins + self.padding_idx = config.pad_token_id + self.max_source_positions = config.max_source_positions + self.embed_scale = 1.0 + + self.conv1 = nn.Conv1D(self.num_mel_bins, embed_dim, kernel_size=3, padding=1) + self.conv2 = nn.Conv1D(embed_dim, embed_dim, kernel_size=3, stride=2, padding=1) + + self.embed_positions = nn.Embedding(self.max_source_positions, embed_dim) + + self.layers = nn.ModuleList([EncoderLayer(config) for _ in range(config.encoder_layers)]) + self.layer_norm = nn.LayerNorm(config.d_model) + + def forward(self, input_features: nn.Tensor) -> nn.Tensor: + inputs_embeds = nn.gelu(self.conv1(input_features)) + inputs_embeds = nn.gelu(self.conv2(inputs_embeds)) + + inputs_embeds = nn.permute_dims(inputs_embeds, [0, 2, 1]) + embed_pos = self.embed_positions.weight + + hidden_states = inputs_embeds + embed_pos + + for idx, encoder_layer in enumerate(self.layers): + hidden_states = encoder_layer(hidden_states) + + hidden_states = self.layer_norm(hidden_states) + + return hidden_states + + +class WhisperDecoder(nn.Module): + def __init__(self, config: WhisperConfig): + super().__init__() + + self.max_target_positions = config.max_target_positions + self.max_source_positions = config.max_source_positions + self.embed_scale = 1.0 + self.embed_tokens = nn.Embedding(config.vocab_size, config.d_model) + self.embed_positions = WhisperPositionalEmbedding(self.max_target_positions, config.d_model) + + self.layers = nn.ModuleList([DecoderLayer(config) for _ in range(config.decoder_layers)]) + + self.layer_norm = nn.LayerNorm(config.d_model) + + def forward( + self, + input_ids: nn.Tensor, + total_seq_len: Optional[tir.Var] = None, + encoder_hidden_states: Optional[nn.Tensor] = None, + cached_encoder_key_value: Optional[Tuple[Tuple[nn.Tensor]]] = None, + attention_mask: Optional[nn.Tensor] = None, + ): + # total_seq_len = Length of generated tokens + input_embeds = self.embed_tokens(input_ids) + past_seq_len = total_seq_len - 1 + position_embeds = self.embed_positions(input_ids, offset=past_seq_len) + + hidden_states = input_embeds + position_embeds + + all_encoder_key_value = () + for idx, decoder_layer in enumerate(self.layers): + ith_cached_encoder_key_value = ( + cached_encoder_key_value[idx] if cached_encoder_key_value is not None else None + ) + hidden_states, encoder_key_value = decoder_layer( + hidden_states=hidden_states, + total_seq_len=total_seq_len, + encoder_hidden_states=encoder_hidden_states, + cached_encoder_hidden_states=ith_cached_encoder_key_value, + attention_mask=attention_mask, + ) + if cached_encoder_key_value is None: + all_encoder_key_value += (encoder_key_value,) + + hidden_states = self.layer_norm(hidden_states) + + if cached_encoder_key_value is None: + return hidden_states, all_encoder_key_value + else: + return hidden_states, None + + +class WhisperModel(nn.Module): + def __init__(self, config: WhisperConfig): + self.encoder = WhisperEncoder(config) + self.decoder = WhisperDecoder(config) + + +class WhisperForConditionalGeneration(nn.Module): + def __init__(self, config: WhisperConfig): + self.config = config + self.model = WhisperModel(config) + self.proj_out = nn.Linear(config.d_model, config.vocab_size, bias=False) + + def encode(self, input_features: nn.Tensor) -> nn.Tensor: + return self.model.encoder(input_features) + + def decode( + self, input_ids: nn.Tensor, total_seq_len: int, encoder_hidden_states: nn.Tensor + ) -> Tuple[nn.Tensor, Tuple[Tuple[nn.Tensor]]]: + hidden_states, all_encoder_key_value = self.model.decoder.forward( + input_ids=input_ids, + total_seq_len=total_seq_len, + encoder_hidden_states=encoder_hidden_states, + cached_encoder_key_value=None, + attention_mask=None, + ) + lm_logits = self.proj_out(hidden_states) + return lm_logits, all_encoder_key_value + + def prefill( + self, + input_ids: nn.Tensor, + total_seq_len: int, + cached_encoder_key_value: Tuple[Tuple[nn.Tensor]], + ) -> nn.Tensor: + hidden_states, _ = self.model.decoder.forward( + input_ids=input_ids, + total_seq_len=total_seq_len, + encoder_hidden_states=None, + cached_encoder_key_value=cached_encoder_key_value, + attention_mask=None, + ) + lm_logits = self.proj_out(hidden_states) + return lm_logits + + def get_default_spec(self): + batch_size = 1 + encode_input_ndim = 16000 * 30 // 160 + mod_spec = { + "encode": { + "input_features": nn.spec.Tensor( + [batch_size, self.config.num_mel_bins, encode_input_ndim], self.dtype + ), + "$": { + "param_mode": "packed", + "effect_mode": "packed", + }, + }, + "decode": { + "input_ids": nn.spec.Tensor([batch_size, "seq_len"], "int32"), + "total_seq_len": int, + "encoder_hidden_states": nn.spec.Tensor( + [batch_size, self.config.max_source_positions, self.config.d_model], self.dtype + ), + "$": { + "param_mode": "packed", + "effect_mode": "packed", + }, + }, + "prefill": { + "input_ids": nn.spec.Tensor([batch_size, 1], "int32"), + "total_seq_len": int, + "cached_encoder_key_value": tuple( + tuple( + nn.spec.Tensor( + [ + 1, + self.config.max_source_positions, + self.config.decoder_attention_heads, + self.config.d_model // self.config.decoder_attention_heads, + ], + self.dtype, + ) + for i2 in range(2) + ) + for i1 in range(self.config.num_hidden_layers) + ), + "$": { + "param_mode": "packed", + "effect_mode": "packed", + }, + }, + } + return nn.spec.ModuleSpec.from_raw(mod_spec, self) diff --git a/python/mlc_chat/model/whisper/whisper_quantization.py b/python/mlc_chat/model/whisper/whisper_quantization.py new file mode 100644 index 0000000000..1aafbe1082 --- /dev/null +++ b/python/mlc_chat/model/whisper/whisper_quantization.py @@ -0,0 +1,40 @@ +"""This file specifies how MLC's Whisper parameters are quantized using group quantization +or other formats.""" + +from typing import Tuple + +from tvm.relax.frontend import nn + +from mlc_chat.loader import QuantizeMapping +from mlc_chat.quantization import GroupQuantize, NoQuantize + +from .whisper_model import WhisperConfig, WhisperForConditionalGeneration + + +def group_quant( + model_config: WhisperConfig, + quantization: GroupQuantize, +) -> Tuple[nn.Module, QuantizeMapping]: + """Quantize a GPTNeoX-architecture model using group quantization.""" + model: nn.Module = WhisperForConditionalGeneration(model_config) + model.to(quantization.model_dtype) + model.dtype = quantization.model_dtype + quant_map = QuantizeMapping({}, {}) + model = quantization.quantize_model( + model, + quant_map, + "", + ) + return model, quant_map + + +def no_quant( + model_config: WhisperConfig, + quantization: NoQuantize, +) -> Tuple[nn.Module, QuantizeMapping]: + """Quantize a GPTNeoX model without quantization.""" + model: nn.Module = WhisperForConditionalGeneration(model_config) + model.to(quantization.model_dtype) + model.dtype = quantization.model_dtype + quant_map = QuantizeMapping({}, {}) + return model, quant_map diff --git a/python/mlc_chat/quantization/group_quantization.py b/python/mlc_chat/quantization/group_quantization.py index d854fd9cef..872aa49e9d 100644 --- a/python/mlc_chat/quantization/group_quantization.py +++ b/python/mlc_chat/quantization/group_quantization.py @@ -1,4 +1,5 @@ """The group quantization config""" + from dataclasses import dataclass from functools import partial from typing import Any, Callable, List, Literal, Optional, Tuple, Union @@ -394,19 +395,25 @@ def forward(self, x: nn.Tensor) -> nn.Tensor: # pylint: disable=invalid-name weight, scale, axis=self.config.linear_quant_axis, - out_shape=[ - tir.IntImm("int64", self.out_features) - if isinstance(self.out_features, int) - else weight.shape[0], # Reuse same tir.Var for symbolic shape (after Exporter) - tir.IntImm("int64", self.in_features), - ] - if self.config.linear_weight_layout == "NK" - else [ - tir.IntImm("int64", self.in_features), - tir.IntImm("int64", self.out_features) - if isinstance(self.out_features, int) - else weight.shape[1], # Reuse same tir.Var for symbolic shape (after Exporter) - ], + out_shape=( + [ + ( + tir.IntImm("int64", self.out_features) + if isinstance(self.out_features, int) + else weight.shape[0] + ), # Reuse same tir.Var for symbolic shape (after Exporter) + tir.IntImm("int64", self.in_features), + ] + if self.config.linear_weight_layout == "NK" + else [ + tir.IntImm("int64", self.in_features), + ( + tir.IntImm("int64", self.out_features) + if isinstance(self.out_features, int) + else weight.shape[1] + ), # Reuse same tir.Var for symbolic shape (after Exporter) + ] + ), ), name_hint="dequantize", args=[self.q_weight, self.q_scale], @@ -485,9 +492,11 @@ def forward(self, x: nn.Tensor): # pylint: disable=invalid-name scale, axis=-1, out_shape=[ - tir.IntImm("int64", self.num) - if isinstance(self.num, int) - else weight.shape[0], # Reuse same tir.Var for symbolic shape (after Exporter) + ( + tir.IntImm("int64", self.num) + if isinstance(self.num, int) + else weight.shape[0] + ), # Reuse same tir.Var for symbolic shape (after Exporter) tir.IntImm("int64", self.dim), ], ), diff --git a/test_cli.py b/test_cli.py new file mode 100644 index 0000000000..f0313bddab --- /dev/null +++ b/test_cli.py @@ -0,0 +1,32 @@ +help_str = """You can use the following special commands: + /help print the special commands + /exit quit the cli + /stats print out the latest stats (token/sec) + /reset restart a fresh chat + /reload [model] reload model `model` from disk, or reload the current \ +model if `model` is not specified +""" +print(help_str) + +from prompt_toolkit import prompt +from prompt_toolkit.key_binding import KeyBindings + +kb = KeyBindings() + + +@kb.add("escape", "enter") +def _(event): + event.current_buffer.insert_text("\n") + + +@kb.add("enter") +def _(event): + event.current_buffer.validate_and_handle() + + +result = prompt(">", key_bindings=kb, multiline=True) +print("12") +print(result) + +# user_input = get_multiline_input() +# print(user_input) diff --git a/test_disco.py b/test_disco.py new file mode 100644 index 0000000000..292fcc12c5 --- /dev/null +++ b/test_disco.py @@ -0,0 +1,37 @@ +# pylint: skip-file + +MODEL = "HF://junrushao/Llama-2-7b-chat-hf-q4f16_1-MLC" +# MODEL = "HF://junrushao/RedPajama-INCITE-Chat-3B-v1-q4f16_1-MLC" +# MODEL = "HF://junrushao/WizardCoder-15B-V1.0-q4f16_1-MLC" +# MODEL = "HF://junrushao/Mistral-7B-Instruct-v0.2-q4f16_1-MLC" +# MODEL = "HF://junrushao/phi-1_5-q4f16_1-MLC" +# MODEL = "HF://junrushao/phi-2-q4f16_1-MLC" +TP_SHARDS = 2 +import os + +from mlc_chat import ChatConfig, ChatModule, callback +from mlc_chat.support import logging + +# temp_dir = "/opt/scratch/lesheng/mlc-llm/dist/tmp" + +# os.environ["MLC_TEMP_DIR"] = temp_dir +# os.environ["MLC_CACHE_DIR"] = temp_dir +# os.environ["MLC_JIT_POLICY"] = "REDO" + + +logging.enable_logging() + +cm = ChatModule( + MODEL, + device="cuda", + chat_config=ChatConfig( + # context_window_size=1024, + # prefill_chunk_size=1024 if "Mistral" not in MODEL else 4096, + tensor_parallel_shards=TP_SHARDS, + # opt="flashinfer=0;cublas_gemm=1;cudagraph=0", + ), +) +cm.generate( + "What is the meaning of life?", + progress_callback=callback.StreamToStdout(callback_interval=2), +) diff --git a/test_disco_batched.py b/test_disco_batched.py new file mode 100644 index 0000000000..0425e39788 --- /dev/null +++ b/test_disco_batched.py @@ -0,0 +1,85 @@ +# pylint: skip-file +import json +import logging +import os +import shutil + +import tvm + +from mlc_chat import ChatConfig, ChatModule + +temp_dir = "/opt/scratch/lesheng/mlc-llm/dist/disco_tmp" + +os.environ["MLC_TEMP_DIR"] = temp_dir +os.environ["MLC_CACHE_DIR"] = temp_dir + +logging.basicConfig( + level=logging.INFO, + style="{", + datefmt="%Y-%m-%d %H:%M:%S", + format="[{asctime}] {levelname} {filename}:{lineno}: {message}", +) + + +def test_model(url, num_shards=1): + try: + cm = ChatModule( + url, + chat_config=ChatConfig( + context_window_size=1024, + max_gen_len=200, + tensor_parallel_shards=num_shards, + ), + device="cuda", + ) + output = cm.generate( + # prompt="What is the meaning of life?", + prompt="Write a python program that computes a + b." + ) + except: + output = "Failed" + return output + + +print(test_model("HF://junrushao/gpt2-q4f16_1-MLC", 4)) +exit(0) + +with open("model_info.json", "r") as f: + model_dict = json.load(f) + +testout_dir = "disco_out" +os.makedirs(testout_dir, exist_ok=True) + +# shutil.rmtree(temp_dir, ignore_errors=True) +for model in model_dict: + model_infos = model_dict[model] + for quant in ["q4f16_1", "q4f32_1"]: # , "q3f16_1"]: + url = model_infos[quant] + if len(url) == 0: + continue + + print(f"{model}/{quant}: {url}") + + os.makedirs(temp_dir, exist_ok=True) + + model_out_dir = os.path.join(testout_dir, model) + os.makedirs(model_out_dir, exist_ok=True) + for i in [1, 2, 4]: # , 8]: + print(i) + testout_file = os.path.join(model_out_dir, f"{quant}-{i}.out") + + if os.path.exists(testout_file): + print(f"skip {testout_file}") + continue + + output = test_model(url, i) + with open(testout_file, "w+") as f: + f.write(output + "\n") + + shutil.rmtree(temp_dir, ignore_errors=True) + + # if f"{model}_{quant}" in ["CodeLlama-7b-hf_q4f32_1"]: + # continue + # if os.path.exists(testout_file): + # print(f"skip {testout_file}") + # continue diff --git a/test_gpt.sh b/test_gpt.sh new file mode 100755 index 0000000000..febd080f9b --- /dev/null +++ b/test_gpt.sh @@ -0,0 +1,22 @@ +#!/bin/bash +set -e + +# QUANT=q0f16 +QUANT=q4f16_1 +NUM_SHARDS=$1 + +MODEL_PATH=/opt/scratch/lesheng/models/gpt2-medium +OUTPUT_PATH=./dist/GPT-${QUANT}/ + +mkdir -p $OUTPUT_PATH + +python -m mlc_chat gen_config $MODEL_PATH/config.json --quantization $QUANT --conv-template phi-2 -o $OUTPUT_PATH/params --tensor-parallel-shards $NUM_SHARDS + +python -m mlc_chat compile $OUTPUT_PATH/params -o $OUTPUT_PATH/model.so --debug-dump $OUTPUT_PATH/debug + +# python -m mlc_chat convert_weight $MODEL_PATH --quantization $QUANT -o $OUTPUT_PATH/params + +#"What is the meaning of life?" +# CUDA_VISIBLE_DEVICES=4,5,6,7 + +python -m mlc_chat.cli.benchmark --model $OUTPUT_PATH/params --model-lib $OUTPUT_PATH/model.so --device "cuda:0" --prompt "a + b" --generate-length 256 diff --git a/test_llama.sh b/test_llama.sh new file mode 100755 index 0000000000..1361365181 --- /dev/null +++ b/test_llama.sh @@ -0,0 +1,20 @@ +#!/bin/bash +set -e + +# QUANT=q0f16 +QUANT=q4f16_1 +# QUANT=q4f16_0 +# QUANT=q4f16_ft +MODEL_PATH=/opt/models/llama-2/llama-2-7b-chat-hf +OUTPUT_PATH=./dist/new-llama-${QUANT}/ +NUM_SHARDS=2 + +mkdir -p $OUTPUT_PATH + +# python -m mlc_chat gen_config $MODEL_PATH/config.json --quantization $QUANT --conv-template llama-2 -o $OUTPUT_PATH/params --tensor-parallel-shards $NUM_SHARDS + +# python -m mlc_chat compile $OUTPUT_PATH/params -o $OUTPUT_PATH/model.so --opt "O2" + +# PRESHARD_NUM=$NUM_SHARDS python -m mlc_chat convert_weight $MODEL_PATH --quantization $QUANT -o $OUTPUT_PATH/params + +CUDA_VISIBLE_DEVICES=4,5,6,7 python -m mlc_chat.cli.benchmark --model $OUTPUT_PATH/params --model-lib $OUTPUT_PATH/model.so --device "cuda:0" --prompt "What is the meaning of life?" --generate-length 256 --num-shards $NUM_SHARDS diff --git a/test_mistral.sh b/test_mistral.sh new file mode 100755 index 0000000000..972480ea6c --- /dev/null +++ b/test_mistral.sh @@ -0,0 +1,22 @@ +#!/bin/bash +set -e + +# QUANT=q0f16 +QUANT=q4f16_1 +MODEL_PATH=/opt/scratch/lesheng/models/Mistral-7B-Instruct-v0.2 +OUTPUT_PATH=./dist/mistral-${QUANT}/ + +mkdir -p $OUTPUT_PATH + +# python -m mlc_chat gen_config $MODEL_PATH/config.json --quantization $QUANT --conv-template mistral_default -o $OUTPUT_PATH/params + +# python -m mlc_chat compile $OUTPUT_PATH/params -o $OUTPUT_PATH/model.so --opt "flashinfer=0;cublas_gemm=1;cudagraph=0" --overrides "context_window_size=32768;sliding_window_size=4096;prefill_chunk_size=4096;attention_sink_size=4" + +# python -m mlc_chat convert_weight $MODEL_PATH --quantization $QUANT -o $OUTPUT_PATH/params + +# CUDA_VISIBLE_DEVICES=6,7 python -m mlc_chat.cli.benchmark --model $OUTPUT_PATH/params --model-lib $OUTPUT_PATH/model.so --device "cuda:0" --prompt "What is the meaning of life?" --generate-length 256 + +CUDA_VISIBLE_DEVICES=6,7 python -m mlc_chat.cli.benchmark \ + --model /opt/scratch/lesheng/mlc-llm/dist/tmp/model_weights/junrushao/Mistral-7B-Instruct-v0.2-q4f16_1-MLC \ + --model-lib /opt/scratch/lesheng/mlc-llm/dist/tmp/model_lib/a4e842ad2814e4e95c171a2685bdbab7.so \ + --device "cuda:0" --prompt "What is the meaning of life?" --generate-length 256 diff --git a/test_multi_gpu.sh b/test_multi_gpu.sh new file mode 100755 index 0000000000..20d3b00347 --- /dev/null +++ b/test_multi_gpu.sh @@ -0,0 +1,14 @@ +#!/bin/bash +set -e + +NUM_SHARDS=2 +MODEL_PATH=/opt/models/llama-2/llama-2-7b-chat-hf/ +OUTPUT_PATH=./dist/new-llama-2-GPU/ + +# python -m mlc_chat gen_config $MODEL_PATH/config.json --quantization q4f16_1 --conv-template llama-2 --context-window-size 4096 -o $OUTPUT_PATH --tensor-parallel-shards $NUM_SHARDS + +# python -m mlc_chat compile $OUTPUT_PATH -o $OUTPUT_PATH/llama.so + +python -m mlc_chat convert_weight $MODEL_PATH --quantization q4f16_1 -o $OUTPUT_PATH + +CUDA_VISIBLE_DEVICES=4,5,6,7 python -m mlc_chat.cli.benchmark --model $OUTPUT_PATH --model-lib $OUTPUT_PATH/llama.so --device "cuda:0" --prompt "What is the meaning of life?" --generate-length 256 diff --git a/test_old.sh b/test_old.sh new file mode 100755 index 0000000000..adcb99f87c --- /dev/null +++ b/test_old.sh @@ -0,0 +1,5 @@ +MODEL_PATH=/opt/models/llama-2/llama-2-7b-chat-hf/ + +CUDA_VISIBLE_DEVICES=4,5,6,7 python build.py --model $MODEL_PATH --quantization q4f16_1 --num-shards 2 --build-model-only --max-seq-len 4096 --use-cache 0 + +CUDA_VISIBLE_DEVICES=4,5,6,7 python build.py --model $MODEL_PATH --quantization q4f16_1 --num-shards 2 --convert-weight-only --max-seq-len 4096 --use-cache 0 diff --git a/test_phi.sh b/test_phi.sh new file mode 100755 index 0000000000..076d0381ba --- /dev/null +++ b/test_phi.sh @@ -0,0 +1,30 @@ +#!/bin/bash +set -e + +# QUANT=q0f16 +QUANT=q4f16_1 +NUM_SHARDS=$1 + +MODEL_PATH=/opt/scratch/lesheng/phi-2 +OUTPUT_PATH=./dist/phi-2-${QUANT}-${NUM_SHARDS}/ + +# MODEL_PATH=/opt/scratch/lesheng/models/phi-1_5 +# OUTPUT_PATH=./dist/phi-1_5-${QUANT}/ + +# MODEL_PATH=/opt/scratch/lesheng/models/phi-1 +# OUTPUT_PATH=./dist/phi-1-${QUANT}/ + +mkdir -p $OUTPUT_PATH + +python -m mlc_chat gen_config $MODEL_PATH/config.json --quantization $QUANT --conv-template phi-2 -o $OUTPUT_PATH/params --tensor-parallel-shards $NUM_SHARDS + +python -m mlc_chat compile $OUTPUT_PATH/params -o $OUTPUT_PATH/model.so --debug-dump $OUTPUT_PATH/debug + +# python -m mlc_chat convert_weight $MODEL_PATH --quantization $QUANT -o $OUTPUT_PATH/params + +#"What is the meaning of life?" +# CUDA_VISIBLE_DEVICES=4,5,6,7 + +python -m mlc_chat.cli.benchmark --model $OUTPUT_PATH/params --model-lib $OUTPUT_PATH/model.so --device "cuda:0" --prompt "Print a + b" --generate-length 256 + +# python -m mlc_chat.cli.benchmark --model ./dist/phi-2-q4f16_1-4/params --model-lib $OUTPUT_PATH/model.so --device "cuda:0" --prompt "Print a + b" --generate-length 256 diff --git a/test_red_pajama.sh b/test_red_pajama.sh new file mode 100755 index 0000000000..fbbaab4718 --- /dev/null +++ b/test_red_pajama.sh @@ -0,0 +1,20 @@ +#!/bin/bash +set -e + +# QUANT=q0f16 +QUANT=q4f16_1 +MODEL_PATH=/opt/scratch/lesheng/models/RedPajama-INCITE-Chat-3B-v1 +OUTPUT_PATH=./dist/new-red-pajama-${QUANT}/ + +mkdir -p $OUTPUT_PATH + +# python -m mlc_chat gen_config $MODEL_PATH/config.json --quantization $QUANT --conv-template redpajama_chat -o $OUTPUT_PATH/params --tensor-parallel-shards 4 + +# python -m mlc_chat compile $OUTPUT_PATH/params -o $OUTPUT_PATH/red-pajama.so --debug-dump $OUTPUT_PATH/debug + +# python -m mlc_chat convert_weight $MODEL_PATH --quantization $QUANT -o $OUTPUT_PATH/params + +# CUDA_VISIBLE_DEVICES=4,5,6,7 +python -m mlc_chat.cli.benchmark --model $OUTPUT_PATH/params --model-lib $OUTPUT_PATH/red-pajama.so --device "cuda:0" --prompt "What is the meaning of life?" --generate-length 256 # --tensor-parallel-shards 2 + +# python -m mlc_chat.cli.benchmark --model ./dist/new-red-pajama-q4f16_1/params --model-lib ./dist/new-red-pajama-q4f16_1/red-pajama.so --device "cuda:0" --prompt "What is the meaning of life?" --generate-length 256 diff --git a/test_slm.py b/test_slm.py new file mode 100644 index 0000000000..35c8ca2ba4 --- /dev/null +++ b/test_slm.py @@ -0,0 +1,91 @@ +import argparse +import os +import pickle + +import numpy as np +import torch +import tvm +from mlc_llm import utils +from transformers import AutoTokenizer +from tvm import relax + + +def load_params(artifact_path: str, device, param_names): + from tvm.contrib import tvmjs # pylint: disable=import-outside-toplevel + + params, meta = tvmjs.load_ndarray_cache(artifact_path, device) + plist = [] + size = meta["ParamSize"] + for pname in param_names: + plist.append(params[pname]) + return plist + + +class TestState: + def __init__(self, args): + self.primary_device = tvm.device(args.primary_device) + ex = tvm.runtime.load_module(args.model_lib) + self.vm = relax.VirtualMachine(ex, self.primary_device) + self.sess = None + + +def deploy_to_pipeline(args) -> None: + state = TestState(args) + x = state.vm["_metadata"]() + metadata = eval(x) + params = metadata["params"] + param_names = [] + for param in params: + param_names.append(param["name"]) + + primary_device = tvm.device(args.primary_device) + const_params = load_params(args.params, primary_device, param_names) + tokenizer = AutoTokenizer.from_pretrained(args.params, trust_remote_code=True) + + print("Tokenizing...") + inputs = tokenizer(args.prompt, return_tensors="pt").input_ids.to(torch.int32).numpy() + print(inputs) + first_sampled_token = tvm.nd.array(np.array([[6234]]).astype("int32"), primary_device) + seq_len_shape = tvm.runtime.ShapeTuple([inputs.shape[1]]) + second_seq_len_shape = tvm.runtime.ShapeTuple([inputs.shape[1] + 1]) + kv_caches = state.vm["_initialize_effect"]() + + print("Running inference...") + print("======================= Starts Encoding =======================") + + prefill_func = state.vm["prefill"] + + inputs = tvm.nd.array(inputs, device=primary_device) + logits, kv_caches = prefill_func(inputs, seq_len_shape, kv_caches, const_params) + print(logits) + + exit(0) + print("======================= Starts Decoding =======================") + logits, kv_caches = state.vm["decode"]( + first_sampled_token, second_seq_len_shape, kv_caches, const_params + ) + + +def _parse_args(): + args = argparse.ArgumentParser() + args.add_argument("--model-lib", type=str, required=True) + args.add_argument("--params", type=str, required=True) + args.add_argument("--primary-device", type=str, default="auto") + args.add_argument("--prompt", type=str, default="What is the meaning of life?") + args.add_argument("--time-eval", default=False, action="store_true") + args.add_argument("--skip-rounds", type=int, default=0) + parsed = args.parse_args() + + if parsed.primary_device == "auto": + if tvm.cuda().exist: + parsed.primary_device = "cuda" + elif tvm.metal().exist: + parsed.primary_device = "metal" + else: + raise ValueError("Cannot auto deduce device-name, please set it") + return parsed + + +if __name__ == "__main__": + args = _parse_args() + deploy_to_pipeline(args) diff --git a/test_slm_disco.py b/test_slm_disco.py new file mode 100644 index 0000000000..849ae3ec2e --- /dev/null +++ b/test_slm_disco.py @@ -0,0 +1,132 @@ +# pylint: skip-file +import argparse +import json +import os + +import numpy as np +import torch +import tvm +from transformers import AutoTokenizer +from tvm import relax +from tvm.relax.frontend.nn import IOEffect +from tvm.runtime import disco as di + +import mlc_chat + + +def load_params(artifact_path: str, device, param_names): + from tvm.contrib import tvmjs # pylint: disable=import-outside-toplevel + + params, meta = tvmjs.load_ndarray_cache(artifact_path, device) + plist = [] + size = meta["ParamSize"] + for pname in param_names: + plist.append(params[pname]) + return plist + + +def CopyToWorker0(sess: di.ProcessSession, host_array): + array = sess.empty(host_array.shape, host_array.dtype) + sess.copy_to_worker_0(host_array, array) + return array + + +def deploy_to_pipeline(args) -> None: + model_config = open(os.path.join(args.params, "mlc-chat-config.json")).read() + + devices = list(range(args.tensor_parallel_shards)) + sess = di.ProcessSession( + num_workers=args.tensor_parallel_shards, entrypoint="mlc_chat.cli.worker" + ) + sess.init_ccl("nccl", *devices) + print(args.model_lib) + mod = sess.load_vm_module(args.model_lib) + create_kv_cache = mod["_initialize_effect"] + reset_kv = sess.get_global_func("vm.builtin.attention_kv_cache_array_clear") + + prefill_func = mod["prefill"] + + kv_caches = create_kv_cache() + + # local_kv = kv_caches.debug_get_from_remote(0) + # print(type(local_kv)) + # print(len(local_kv)) + # exit(0) + + loader = sess.get_global_func("mlc.loader.LoadMultiGPU") + params = loader(args.params, mod, model_config) + + input_tokens_decode = sess.empty((1, 1), "int32") + + tokenizer = AutoTokenizer.from_pretrained(args.params, trust_remote_code=True) + prompt_tokens = tokenizer(args.prompt, return_tensors="pt").input_ids.to(torch.int32).numpy() + print(prompt_tokens) + total_seq_len = prompt_tokens.shape[1] + prompt_tokens = tvm.nd.array(prompt_tokens) + + sess.sync_worker_0() + # ForwardTokens + input_data = CopyToWorker0(sess, prompt_tokens) + seq_len_shape = tvm.runtime.ShapeTuple([total_seq_len]) + ret = prefill_func(input_data, seq_len_shape, kv_caches, params) + + logits_on_devices = ret.debug_get_from_remote(0)[0] + print(logits_on_devices) + total_seq_len += 1 + exit(0) + second_seq_len_shape = tvm.runtime.ShapeTuple([inputs.shape[1] + 1]) + + print("Running inference...") + print("======================= Starts Encoding =======================") + + try: + prefill_func = state.vm["prefill"] + except AttributeError: + prefill_func = None + + if inputs.shape[1] > 1 and prefill_func: + inputs = tvm.nd.array(inputs, device=primary_device) + logits, kv_caches = prefill_func(inputs, seq_len_shape, kv_caches, const_params) + else: + for i in range(inputs.shape[1]): + input_slice = tvm.nd.array(inputs[:, i : i + 1], device=primary_device) + logits, kv_caches = state.vm["decode"]( + input_slice, seq_len_shape, kv_caches, const_params + ) + + exit(0) + print("======================= Starts Decoding =======================") + logits, kv_caches = state.vm["decode"]( + first_sampled_token, second_seq_len_shape, kv_caches, const_params + ) + + +def _parse_args(): + # @tvm.register_func("vm.builtin.debug_print") + # def _print(lineo: str, array) -> None: + # print(f"{lineo}: shape = {array.shape}, dtype = {array.dtype}, data =\n{array}") + + args = argparse.ArgumentParser() + args.add_argument("--model-lib", type=str, required=True) + args.add_argument("--params", type=str, required=True) + args.add_argument("--tensor-parallel-shards", type=int, default=4) + args.add_argument("--primary-device", type=str, default="auto") + args.add_argument("--prompt", type=str, default="What is the meaning of life?") + args.add_argument("--time-eval", default=False, action="store_true") + args.add_argument("--skip-rounds", type=int, default=0) + parsed = args.parse_args() + + if parsed.primary_device == "auto": + if tvm.cuda().exist: + parsed.primary_device = "cuda" + elif tvm.metal().exist: + parsed.primary_device = "metal" + else: + raise ValueError("Cannot auto deduce device-name, please set it") + return parsed + + +if __name__ == "__main__": + args = _parse_args() + deploy_to_pipeline(args) + deploy_to_pipeline(args) diff --git a/test_whisper.py b/test_whisper.py new file mode 100644 index 0000000000..0dc9593cf4 --- /dev/null +++ b/test_whisper.py @@ -0,0 +1,205 @@ +# pylint: disable=invalid-name,missing-docstring +import json +import time +from typing import Any, Tuple + +import numpy as np +import torch +import tvm +from datasets import load_dataset +from scipy.io import wavfile +from transformers import WhisperForConditionalGeneration as hf_Whisper +from transformers import WhisperProcessor +from tvm import relax + +from mlc_chat.model.whisper.whisper_model import ( + WhisperConfig, + WhisperForConditionalGeneration, +) + + +def load_params(artifact_path: str, device, param_names): + from tvm.contrib import tvmjs # pylint: disable=import-outside-toplevel + + params, meta = tvmjs.load_ndarray_cache(artifact_path, device) + plist = [] + size = meta["ParamSize"] + for pname in param_names: + plist.append(params[pname]) + return plist + + +def get_param_list(vm): + x = vm["_metadata"]() + metadata = eval(x) + params = metadata["params"] + param_names = [] + for param in params: + param_names.append(param["name"]) + return param_names + + +def get_param_list000(config, dtype, submodel_name): + model = WhisperForConditionalGeneration(config) + # model.to(dtype) + encode_params = [] + decode_params = [] + prefill_params = [] + for param, _ in model.named_parameters(): + if param.startswith("model.encoder"): + encode_params.append(param) + else: + encode_params.append(param) + decode_params.append(param) + if "encoder_attn_k_proj" in param: + continue + if "encoder_attn_v_proj" in param: + continue + prefill_params.append(param) + + return encode_params, decode_params, prefill_params + + +def load_whisper_from_hf(model_name="openai/whisper-medium") -> Tuple[hf_Whisper, WhisperProcessor]: + processor = WhisperProcessor.from_pretrained(model_name) + # hf_model = hf_Whisper.from_pretrained(model_name) + # hf_model = hf_model.eval().to("cuda") + hf_model = None + return hf_model, processor + + +def load_data(processor: WhisperProcessor, test_idx: int) -> Tuple[torch.Tensor, str]: + samplerate, data = wavfile.read("../librispeech_dummy.wav") + # print(samplerate) + # print(len(data)) + input_features = processor( + data, sampling_rate=samplerate, return_tensors="pt" + ).input_features.to("cuda") + # print("input_features: ", input_features.shape) + return input_features, None + + +def pipe( + model: Any, const_params: Any, config: WhisperConfig, input_features, device +) -> torch.Tensor: + kv_caches = model["_initialize_effect"]() + # encode + encode_start = time.time() + encode_output, kv_caches = model["encode"](input_features, kv_caches, const_params) + device.sync() + encode_end = time.time() + print(f"encode {(encode_end - encode_start) * 1000:.2f}") + + # decode start token + input_ids = torch.tensor([[config.decoder_start_token_id]], dtype=torch.int32) # .to("cuda") + generated_tokens = [config.decoder_start_token_id] + + input_ids = tvm.nd.array(input_ids, tvm.cuda()) + while True: + decode_start = time.time() + if len(generated_tokens) == 1: + (outputs, encode_kv_cache), kv_caches = model["decode"]( + input_ids, + tvm.runtime.ShapeTuple([len(generated_tokens)]), + encode_output, + kv_caches, + const_params, + ) + else: + outputs, kv_caches = model["prefill"]( + input_ids, + tvm.runtime.ShapeTuple([len(generated_tokens)]), + encode_kv_cache, + kv_caches, + const_params, + ) + device.sync() + decode_end = time.time() + print(f"{(decode_end - decode_start) * 1000:.2f}") + + outputs_logits = outputs.numpy() + next_token_logits = outputs_logits[:, 0, :] + + # suppress tokens + next_tokens_scores = next_token_logits + next_tokens_scores[:, config.suppress_tokens] = -float("inf") + + # suppress tokens at begin + if len(generated_tokens) == 1: # + config.forced_decoder_ids[-1][0]: + next_tokens_scores[:, config.begin_suppress_tokens] = -float("inf") + + # force tokens at sepcific position + generation_idx = len(generated_tokens) + current_token = dict(config.forced_decoder_ids).get(generation_idx, None) + if current_token is not None: + next_tokens_scores[:, :] = -float("inf") + next_tokens_scores[:, current_token] = 0 + + # argmax + next_token = np.argmax(next_tokens_scores, axis=-1)[0] + # next_token = torch.argmax(next_tokens_scores, dim=-1)[0] + # input_ids[0][0] = next_token + input_ids = tvm.nd.array(np.array([[next_token]], dtype=np.int32), tvm.cuda()) + + generated_tokens.append(next_token) + + # stop when we meet eos_token_id or exceed the maximum length + if ( + next_token == config.eos_token_id + or len(generated_tokens) == config.max_target_positions + ): + break + + return generated_tokens + + +def main(): + with open("../models/whisper-large-v3/config.json", "r", encoding="utf-8") as config_file: + config = json.load(config_file) + config = WhisperConfig.from_dict(config) + + model_dir = "dist/whisper-q0f16" + # Set the device and target + dev = tvm.cuda() + target = tvm.target.Target( + { + "kind": "cuda", + "max_shared_memory_per_block": dev.max_shared_memory_per_block, + "max_threads_per_block": dev.max_threads_per_block, + "thread_warp_size": dev.warp_size, + "registers_per_block": 65536, + "arch": "sm_" + tvm.cuda().compute_version.replace(".", ""), + } + ) + ex = tvm.runtime.load_module(f"{model_dir}/model.so") + vm = relax.VirtualMachine(ex, dev) + param_list = get_param_list(vm) + const_params = load_params(f"{model_dir}/params", dev, param_list) + + # load model from transformers + hf_model, processor = load_whisper_from_hf("../models/whisper-large-v3/") + + # Test on librispeech_asr_dummy + input_features, text = load_data(processor, test_idx=0) + input_features = input_features.to("cpu").to(torch.float16) + input_features = tvm.nd.array(input_features, device=dev) + + # const_params = [encode_params, decode_params, prefill_params] + generated_tokens = pipe(vm, const_params, config, input_features, dev) + + # # compare with hf whisper output + # hf_predicted_ids = hf_model.generate(input_features).to("cpu") + # assert torch.equal(torch.tensor([generated_tokens], dtype=torch.long), hf_predicted_ids) + + # decode token ids to text + output = processor.decode(generated_tokens, skip_special_tokens=True) + assert ( + output + == " Mr. Quilter is the apostle of the middle classes and we are glad to welcome his gospel." + ) + + print("Transcription:\n", output) + + +if __name__ == "__main__": + main() diff --git a/test_whisper.sh b/test_whisper.sh new file mode 100755 index 0000000000..5698dd996e --- /dev/null +++ b/test_whisper.sh @@ -0,0 +1,21 @@ +#!/bin/bash +set -ex + +QUANT=q0f16 +# QUANT=q4f16_1 +# NUM_SHARDS=$1 + +MODEL_PATH=/opt/scratch/lesheng/models/whisper-large-v3 +OUTPUT_PATH=./dist/whisper-${QUANT}/ + +mkdir -p $OUTPUT_PATH + +# python -m mlc_chat gen_config $MODEL_PATH/config.json --quantization $QUANT --conv-template phi-2 -o $OUTPUT_PATH/params + +python -m mlc_chat compile $OUTPUT_PATH/params -o $OUTPUT_PATH/model.so --debug-dump $OUTPUT_PATH/debug --opt="cublas_gemm=0" + +# python -m mlc_chat convert_weight $MODEL_PATH --quantization $QUANT -o $OUTPUT_PATH/params + +# CUDA_VISIBLE_DEVICES=4,5,6,7 + +# python -m mlc_chat.cli.benchmark --model $OUTPUT_PATH/params --model-lib $OUTPUT_PATH/model.so --device "cuda:0" --prompt "Print a + b" --generate-length 256 diff --git a/test_yi.sh b/test_yi.sh new file mode 100755 index 0000000000..3fd7f35f3b --- /dev/null +++ b/test_yi.sh @@ -0,0 +1,19 @@ +#!/bin/bash +set -e + +# QUANT=q0f16 +QUANT=q4f16_1 +# QUANT=q4f16_0 +MODEL_PATH=../models/Yi-34B +OUTPUT_PATH=./dist/Yi-34B-${QUANT}/ +NUM_SHARDS=1 + +mkdir -p $OUTPUT_PATH + +python -m mlc_chat gen_config $MODEL_PATH/config.json --quantization $QUANT --conv-template llama-2 -o $OUTPUT_PATH/params --tensor-parallel-shards $NUM_SHARDS + +python -m mlc_chat compile $OUTPUT_PATH/params -o $OUTPUT_PATH/model.so --opt "flashinfer=0;cublas_gemm=0;cudagraph=0" + +python -m mlc_chat convert_weight $MODEL_PATH --quantization $QUANT -o $OUTPUT_PATH/params + +CUDA_VISIBLE_DEVICES=4,5,6,7 python -m mlc_chat.cli.benchmark --model $OUTPUT_PATH/params --model-lib $OUTPUT_PATH/model.so --device "cuda:0" --prompt "What is the meaning of life?" --generate-length 256 --num-shards $NUM_SHARDS diff --git a/tests/python/test_model_whisper.py b/tests/python/test_model_whisper.py index 40515954a9..669543392b 100644 --- a/tests/python/test_model_whisper.py +++ b/tests/python/test_model_whisper.py @@ -1,16 +1,17 @@ # pylint: disable=invalid-name,missing-docstring -from typing import Tuple, Any +from typing import Any, Tuple import torch +import tvm +from datasets import load_dataset from transformers import WhisperForConditionalGeneration as hf_Whisper from transformers import WhisperProcessor -from datasets import load_dataset -import torch - -import tvm from tvm.relax.frontend.nn import spec -from mlc_llm.models.whisper import WhisperConfig, WhisperForConditionalGeneration +from mlc_chat.model.whisper.whisper_model import ( + WhisperConfig, + WhisperForConditionalGeneration, +) def load_whisper_from_hf() -> Tuple[hf_Whisper, WhisperProcessor]: diff --git a/wasted/1.py b/wasted/1.py new file mode 100644 index 0000000000..0b50bebc41 --- /dev/null +++ b/wasted/1.py @@ -0,0 +1,4652 @@ +# pylint: skip-file +{ + "model_type": "gpt_neox", + "quantization": "q4f16_1", + "context_window_size": 2048, + "prefill_chunk_size": 2048, + "sliding_window_size": -1, + "attention_sink_size": -1, + "tensor_parallel_shards": 2, + "params": [ + { + "name": "gpt_neox.embed_in.q_weight", + "shape": [50432, 320], + "dtype": "uint32", + "preprocs": [], + }, + { + "name": "gpt_neox.embed_in.q_scale", + "shape": [50432, 80], + "dtype": "float16", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.0.input_layernorm.weight", + "shape": [2560], + "dtype": "float16", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.0.input_layernorm.bias", + "shape": [2560], + "dtype": "float16", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.0.post_attention_layernorm.weight", + "shape": [2560], + "dtype": "float16", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.0.post_attention_layernorm.bias", + "shape": [2560], + "dtype": "float16", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.0.attention.query_key_value.q_weight", + "shape": [3840, 320], + "dtype": "uint32", + "preprocs": [ + { + "func_name": "_shard_qkv_q_weight", + "out_shape": [2, 3840, 320], + "out_dtype": "uint32", + } + ], + }, + { + "name": "gpt_neox.layers.0.attention.query_key_value.q_scale", + "shape": [3840, 80], + "dtype": "float16", + "preprocs": [ + { + "func_name": "_shard_qkv_q_scale", + "out_shape": [2, 3840, 80], + "out_dtype": "float16", + } + ], + }, + { + "name": "gpt_neox.layers.0.attention.query_key_value.bias", + "shape": [3840], + "dtype": "float16", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.0.attention.dense.q_weight", + "shape": [2560, 160], + "dtype": "uint32", + "preprocs": [ + { + "func_name": "_shard_dense_q_weight", + "out_shape": [2, 2560, 160], + "out_dtype": "uint32", + } + ], + }, + { + "name": "gpt_neox.layers.0.attention.dense.q_scale", + "shape": [2560, 40], + "dtype": "float16", + "preprocs": [ + { + "func_name": "_shard_dense_q_scale", + "out_shape": [2, 2560, 40], + "out_dtype": "float16", + } + ], + }, + { + "name": "gpt_neox.layers.0.attention.dense.bias", + "shape": [2560], + "dtype": "float16", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.0.mlp.dense_h_to_4h.q_weight", + "shape": [5120, 320], + "dtype": "uint32", + "preprocs": [ + { + "func": "_shard_dense_h_to_4h_q_weight", + "out_shape": [2, 5120, 320], + "out_dtype": "uint32", + } + ], + }, + { + "name": "gpt_neox.layers.0.mlp.dense_h_to_4h.q_scale", + "shape": [5120, 80], + "dtype": "float16", + "preprocs": [ + { + "func": "_shard_dense_h_to_4h_q_scale", + "out_shape": [2, 5120, 80], + "out_dtype": "float16", + } + ], + }, + { + "name": "gpt_neox.layers.0.mlp.dense_h_to_4h.bias", + "shape": [5120], + "dtype": "float32", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.0.mlp.dense_4h_to_h.q_weight", + "shape": [2560, 640], + "dtype": "uint32", + "preprocs": [ + { + "func_name": "_shard_dense_4h_to_h_q_weight", + "out_shape": [2, 2560, 640], + "out_dtype": "uint32", + } + ], + }, + { + "name": "gpt_neox.layers.0.mlp.dense_4h_to_h.q_scale", + "shape": [2560, 160], + "dtype": "float16", + "preprocs": [ + { + "func_name": "_shard_dense_4h_to_h_q_scale", + "out_shape": [2, 2560, 160], + "out_dtype": "float16", + } + ], + }, + { + "name": "gpt_neox.layers.0.mlp.dense_4h_to_h.bias", + "shape": [2560], + "dtype": "float32", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.1.input_layernorm.weight", + "shape": [2560], + "dtype": "float16", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.1.input_layernorm.bias", + "shape": [2560], + "dtype": "float16", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.1.post_attention_layernorm.weight", + "shape": [2560], + "dtype": "float16", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.1.post_attention_layernorm.bias", + "shape": [2560], + "dtype": "float16", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.1.attention.query_key_value.q_weight", + "shape": [3840, 320], + "dtype": "uint32", + "preprocs": [ + { + "func_name": "_shard_qkv_q_weight", + "out_shape": [2, 3840, 320], + "out_dtype": "uint32", + } + ], + }, + { + "name": "gpt_neox.layers.1.attention.query_key_value.q_scale", + "shape": [3840, 80], + "dtype": "float16", + "preprocs": [ + { + "func_name": "_shard_qkv_q_scale", + "out_shape": [2, 3840, 80], + "out_dtype": "float16", + } + ], + }, + { + "name": "gpt_neox.layers.1.attention.query_key_value.bias", + "shape": [3840], + "dtype": "float16", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.1.attention.dense.q_weight", + "shape": [2560, 160], + "dtype": "uint32", + "preprocs": [ + { + "func_name": "_shard_dense_q_weight", + "out_shape": [2, 2560, 160], + "out_dtype": "uint32", + } + ], + }, + { + "name": "gpt_neox.layers.1.attention.dense.q_scale", + "shape": [2560, 40], + "dtype": "float16", + "preprocs": [ + { + "func_name": "_shard_dense_q_scale", + "out_shape": [2, 2560, 40], + "out_dtype": "float16", + } + ], + }, + { + "name": "gpt_neox.layers.1.attention.dense.bias", + "shape": [2560], + "dtype": "float16", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.1.mlp.dense_h_to_4h.q_weight", + "shape": [5120, 320], + "dtype": "uint32", + "preprocs": [ + { + "func": "_shard_dense_h_to_4h_q_weight", + "out_shape": [2, 5120, 320], + "out_dtype": "uint32", + } + ], + }, + { + "name": "gpt_neox.layers.1.mlp.dense_h_to_4h.q_scale", + "shape": [5120, 80], + "dtype": "float16", + "preprocs": [ + { + "func": "_shard_dense_h_to_4h_q_scale", + "out_shape": [2, 5120, 80], + "out_dtype": "float16", + } + ], + }, + { + "name": "gpt_neox.layers.1.mlp.dense_h_to_4h.bias", + "shape": [5120], + "dtype": "float32", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.1.mlp.dense_4h_to_h.q_weight", + "shape": [2560, 640], + "dtype": "uint32", + "preprocs": [ + { + "func_name": "_shard_dense_4h_to_h_q_weight", + "out_shape": [2, 2560, 640], + "out_dtype": "uint32", + } + ], + }, + { + "name": "gpt_neox.layers.1.mlp.dense_4h_to_h.q_scale", + "shape": [2560, 160], + "dtype": "float16", + "preprocs": [ + { + "func_name": "_shard_dense_4h_to_h_q_scale", + "out_shape": [2, 2560, 160], + "out_dtype": "float16", + } + ], + }, + { + "name": "gpt_neox.layers.1.mlp.dense_4h_to_h.bias", + "shape": [2560], + "dtype": "float32", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.2.input_layernorm.weight", + "shape": [2560], + "dtype": "float16", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.2.input_layernorm.bias", + "shape": [2560], + "dtype": "float16", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.2.post_attention_layernorm.weight", + "shape": [2560], + "dtype": "float16", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.2.post_attention_layernorm.bias", + "shape": [2560], + "dtype": "float16", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.2.attention.query_key_value.q_weight", + "shape": [3840, 320], + "dtype": "uint32", + "preprocs": [ + { + "func_name": "_shard_qkv_q_weight", + "out_shape": [2, 3840, 320], + "out_dtype": "uint32", + } + ], + }, + { + "name": "gpt_neox.layers.2.attention.query_key_value.q_scale", + "shape": [3840, 80], + "dtype": "float16", + "preprocs": [ + { + "func_name": "_shard_qkv_q_scale", + "out_shape": [2, 3840, 80], + "out_dtype": "float16", + } + ], + }, + { + "name": "gpt_neox.layers.2.attention.query_key_value.bias", + "shape": [3840], + "dtype": "float16", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.2.attention.dense.q_weight", + "shape": [2560, 160], + "dtype": "uint32", + "preprocs": [ + { + "func_name": "_shard_dense_q_weight", + "out_shape": [2, 2560, 160], + "out_dtype": "uint32", + } + ], + }, + { + "name": "gpt_neox.layers.2.attention.dense.q_scale", + "shape": [2560, 40], + "dtype": "float16", + "preprocs": [ + { + "func_name": "_shard_dense_q_scale", + "out_shape": [2, 2560, 40], + "out_dtype": "float16", + } + ], + }, + { + "name": "gpt_neox.layers.2.attention.dense.bias", + "shape": [2560], + "dtype": "float16", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.2.mlp.dense_h_to_4h.q_weight", + "shape": [5120, 320], + "dtype": "uint32", + "preprocs": [ + { + "func": "_shard_dense_h_to_4h_q_weight", + "out_shape": [2, 5120, 320], + "out_dtype": "uint32", + } + ], + }, + { + "name": "gpt_neox.layers.2.mlp.dense_h_to_4h.q_scale", + "shape": [5120, 80], + "dtype": "float16", + "preprocs": [ + { + "func": "_shard_dense_h_to_4h_q_scale", + "out_shape": [2, 5120, 80], + "out_dtype": "float16", + } + ], + }, + { + "name": "gpt_neox.layers.2.mlp.dense_h_to_4h.bias", + "shape": [5120], + "dtype": "float32", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.2.mlp.dense_4h_to_h.q_weight", + "shape": [2560, 640], + "dtype": "uint32", + "preprocs": [ + { + "func_name": "_shard_dense_4h_to_h_q_weight", + "out_shape": [2, 2560, 640], + "out_dtype": "uint32", + } + ], + }, + { + "name": "gpt_neox.layers.2.mlp.dense_4h_to_h.q_scale", + "shape": [2560, 160], + "dtype": "float16", + "preprocs": [ + { + "func_name": "_shard_dense_4h_to_h_q_scale", + "out_shape": [2, 2560, 160], + "out_dtype": "float16", + } + ], + }, + { + "name": "gpt_neox.layers.2.mlp.dense_4h_to_h.bias", + "shape": [2560], + "dtype": "float32", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.3.input_layernorm.weight", + "shape": [2560], + "dtype": "float16", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.3.input_layernorm.bias", + "shape": [2560], + "dtype": "float16", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.3.post_attention_layernorm.weight", + "shape": [2560], + "dtype": "float16", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.3.post_attention_layernorm.bias", + "shape": [2560], + "dtype": "float16", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.3.attention.query_key_value.q_weight", + "shape": [3840, 320], + "dtype": "uint32", + "preprocs": [ + { + "func_name": "_shard_qkv_q_weight", + "out_shape": [2, 3840, 320], + "out_dtype": "uint32", + } + ], + }, + { + "name": "gpt_neox.layers.3.attention.query_key_value.q_scale", + "shape": [3840, 80], + "dtype": "float16", + "preprocs": [ + { + "func_name": "_shard_qkv_q_scale", + "out_shape": [2, 3840, 80], + "out_dtype": "float16", + } + ], + }, + { + "name": "gpt_neox.layers.3.attention.query_key_value.bias", + "shape": [3840], + "dtype": "float16", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.3.attention.dense.q_weight", + "shape": [2560, 160], + "dtype": "uint32", + "preprocs": [ + { + "func_name": "_shard_dense_q_weight", + "out_shape": [2, 2560, 160], + "out_dtype": "uint32", + } + ], + }, + { + "name": "gpt_neox.layers.3.attention.dense.q_scale", + "shape": [2560, 40], + "dtype": "float16", + "preprocs": [ + { + "func_name": "_shard_dense_q_scale", + "out_shape": [2, 2560, 40], + "out_dtype": "float16", + } + ], + }, + { + "name": "gpt_neox.layers.3.attention.dense.bias", + "shape": [2560], + "dtype": "float16", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.3.mlp.dense_h_to_4h.q_weight", + "shape": [5120, 320], + "dtype": "uint32", + "preprocs": [ + { + "func": "_shard_dense_h_to_4h_q_weight", + "out_shape": [2, 5120, 320], + "out_dtype": "uint32", + } + ], + }, + { + "name": "gpt_neox.layers.3.mlp.dense_h_to_4h.q_scale", + "shape": [5120, 80], + "dtype": "float16", + "preprocs": [ + { + "func": "_shard_dense_h_to_4h_q_scale", + "out_shape": [2, 5120, 80], + "out_dtype": "float16", + } + ], + }, + { + "name": "gpt_neox.layers.3.mlp.dense_h_to_4h.bias", + "shape": [5120], + "dtype": "float32", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.3.mlp.dense_4h_to_h.q_weight", + "shape": [2560, 640], + "dtype": "uint32", + "preprocs": [ + { + "func_name": "_shard_dense_4h_to_h_q_weight", + "out_shape": [2, 2560, 640], + "out_dtype": "uint32", + } + ], + }, + { + "name": "gpt_neox.layers.3.mlp.dense_4h_to_h.q_scale", + "shape": [2560, 160], + "dtype": "float16", + "preprocs": [ + { + "func_name": "_shard_dense_4h_to_h_q_scale", + "out_shape": [2, 2560, 160], + "out_dtype": "float16", + } + ], + }, + { + "name": "gpt_neox.layers.3.mlp.dense_4h_to_h.bias", + "shape": [2560], + "dtype": "float32", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.4.input_layernorm.weight", + "shape": [2560], + "dtype": "float16", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.4.input_layernorm.bias", + "shape": [2560], + "dtype": "float16", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.4.post_attention_layernorm.weight", + "shape": [2560], + "dtype": "float16", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.4.post_attention_layernorm.bias", + "shape": [2560], + "dtype": "float16", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.4.attention.query_key_value.q_weight", + "shape": [3840, 320], + "dtype": "uint32", + "preprocs": [ + { + "func_name": "_shard_qkv_q_weight", + "out_shape": [2, 3840, 320], + "out_dtype": "uint32", + } + ], + }, + { + "name": "gpt_neox.layers.4.attention.query_key_value.q_scale", + "shape": [3840, 80], + "dtype": "float16", + "preprocs": [ + { + "func_name": "_shard_qkv_q_scale", + "out_shape": [2, 3840, 80], + "out_dtype": "float16", + } + ], + }, + { + "name": "gpt_neox.layers.4.attention.query_key_value.bias", + "shape": [3840], + "dtype": "float16", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.4.attention.dense.q_weight", + "shape": [2560, 160], + "dtype": "uint32", + "preprocs": [ + { + "func_name": "_shard_dense_q_weight", + "out_shape": [2, 2560, 160], + "out_dtype": "uint32", + } + ], + }, + { + "name": "gpt_neox.layers.4.attention.dense.q_scale", + "shape": [2560, 40], + "dtype": "float16", + "preprocs": [ + { + "func_name": "_shard_dense_q_scale", + "out_shape": [2, 2560, 40], + "out_dtype": "float16", + } + ], + }, + { + "name": "gpt_neox.layers.4.attention.dense.bias", + "shape": [2560], + "dtype": "float16", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.4.mlp.dense_h_to_4h.q_weight", + "shape": [5120, 320], + "dtype": "uint32", + "preprocs": [ + { + "func": "_shard_dense_h_to_4h_q_weight", + "out_shape": [2, 5120, 320], + "out_dtype": "uint32", + } + ], + }, + { + "name": "gpt_neox.layers.4.mlp.dense_h_to_4h.q_scale", + "shape": [5120, 80], + "dtype": "float16", + "preprocs": [ + { + "func": "_shard_dense_h_to_4h_q_scale", + "out_shape": [2, 5120, 80], + "out_dtype": "float16", + } + ], + }, + { + "name": "gpt_neox.layers.4.mlp.dense_h_to_4h.bias", + "shape": [5120], + "dtype": "float32", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.4.mlp.dense_4h_to_h.q_weight", + "shape": [2560, 640], + "dtype": "uint32", + "preprocs": [ + { + "func_name": "_shard_dense_4h_to_h_q_weight", + "out_shape": [2, 2560, 640], + "out_dtype": "uint32", + } + ], + }, + { + "name": "gpt_neox.layers.4.mlp.dense_4h_to_h.q_scale", + "shape": [2560, 160], + "dtype": "float16", + "preprocs": [ + { + "func_name": "_shard_dense_4h_to_h_q_scale", + "out_shape": [2, 2560, 160], + "out_dtype": "float16", + } + ], + }, + { + "name": "gpt_neox.layers.4.mlp.dense_4h_to_h.bias", + "shape": [2560], + "dtype": "float32", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.5.input_layernorm.weight", + "shape": [2560], + "dtype": "float16", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.5.input_layernorm.bias", + "shape": [2560], + "dtype": "float16", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.5.post_attention_layernorm.weight", + "shape": [2560], + "dtype": "float16", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.5.post_attention_layernorm.bias", + "shape": [2560], + "dtype": "float16", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.5.attention.query_key_value.q_weight", + "shape": [3840, 320], + "dtype": "uint32", + "preprocs": [ + { + "func_name": "_shard_qkv_q_weight", + "out_shape": [2, 3840, 320], + "out_dtype": "uint32", + } + ], + }, + { + "name": "gpt_neox.layers.5.attention.query_key_value.q_scale", + "shape": [3840, 80], + "dtype": "float16", + "preprocs": [ + { + "func_name": "_shard_qkv_q_scale", + "out_shape": [2, 3840, 80], + "out_dtype": "float16", + } + ], + }, + { + "name": "gpt_neox.layers.5.attention.query_key_value.bias", + "shape": [3840], + "dtype": "float16", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.5.attention.dense.q_weight", + "shape": [2560, 160], + "dtype": "uint32", + "preprocs": [ + { + "func_name": "_shard_dense_q_weight", + "out_shape": [2, 2560, 160], + "out_dtype": "uint32", + } + ], + }, + { + "name": "gpt_neox.layers.5.attention.dense.q_scale", + "shape": [2560, 40], + "dtype": "float16", + "preprocs": [ + { + "func_name": "_shard_dense_q_scale", + "out_shape": [2, 2560, 40], + "out_dtype": "float16", + } + ], + }, + { + "name": "gpt_neox.layers.5.attention.dense.bias", + "shape": [2560], + "dtype": "float16", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.5.mlp.dense_h_to_4h.q_weight", + "shape": [5120, 320], + "dtype": "uint32", + "preprocs": [ + { + "func": "_shard_dense_h_to_4h_q_weight", + "out_shape": [2, 5120, 320], + "out_dtype": "uint32", + } + ], + }, + { + "name": "gpt_neox.layers.5.mlp.dense_h_to_4h.q_scale", + "shape": [5120, 80], + "dtype": "float16", + "preprocs": [ + { + "func": "_shard_dense_h_to_4h_q_scale", + "out_shape": [2, 5120, 80], + "out_dtype": "float16", + } + ], + }, + { + "name": "gpt_neox.layers.5.mlp.dense_h_to_4h.bias", + "shape": [5120], + "dtype": "float32", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.5.mlp.dense_4h_to_h.q_weight", + "shape": [2560, 640], + "dtype": "uint32", + "preprocs": [ + { + "func_name": "_shard_dense_4h_to_h_q_weight", + "out_shape": [2, 2560, 640], + "out_dtype": "uint32", + } + ], + }, + { + "name": "gpt_neox.layers.5.mlp.dense_4h_to_h.q_scale", + "shape": [2560, 160], + "dtype": "float16", + "preprocs": [ + { + "func_name": "_shard_dense_4h_to_h_q_scale", + "out_shape": [2, 2560, 160], + "out_dtype": "float16", + } + ], + }, + { + "name": "gpt_neox.layers.5.mlp.dense_4h_to_h.bias", + "shape": [2560], + "dtype": "float32", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.6.input_layernorm.weight", + "shape": [2560], + "dtype": "float16", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.6.input_layernorm.bias", + "shape": [2560], + "dtype": "float16", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.6.post_attention_layernorm.weight", + "shape": [2560], + "dtype": "float16", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.6.post_attention_layernorm.bias", + "shape": [2560], + "dtype": "float16", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.6.attention.query_key_value.q_weight", + "shape": [3840, 320], + "dtype": "uint32", + "preprocs": [ + { + "func_name": "_shard_qkv_q_weight", + "out_shape": [2, 3840, 320], + "out_dtype": "uint32", + } + ], + }, + { + "name": "gpt_neox.layers.6.attention.query_key_value.q_scale", + "shape": [3840, 80], + "dtype": "float16", + "preprocs": [ + { + "func_name": "_shard_qkv_q_scale", + "out_shape": [2, 3840, 80], + "out_dtype": "float16", + } + ], + }, + { + "name": "gpt_neox.layers.6.attention.query_key_value.bias", + "shape": [3840], + "dtype": "float16", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.6.attention.dense.q_weight", + "shape": [2560, 160], + "dtype": "uint32", + "preprocs": [ + { + "func_name": "_shard_dense_q_weight", + "out_shape": [2, 2560, 160], + "out_dtype": "uint32", + } + ], + }, + { + "name": "gpt_neox.layers.6.attention.dense.q_scale", + "shape": [2560, 40], + "dtype": "float16", + "preprocs": [ + { + "func_name": "_shard_dense_q_scale", + "out_shape": [2, 2560, 40], + "out_dtype": "float16", + } + ], + }, + { + "name": "gpt_neox.layers.6.attention.dense.bias", + "shape": [2560], + "dtype": "float16", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.6.mlp.dense_h_to_4h.q_weight", + "shape": [5120, 320], + "dtype": "uint32", + "preprocs": [ + { + "func": "_shard_dense_h_to_4h_q_weight", + "out_shape": [2, 5120, 320], + "out_dtype": "uint32", + } + ], + }, + { + "name": "gpt_neox.layers.6.mlp.dense_h_to_4h.q_scale", + "shape": [5120, 80], + "dtype": "float16", + "preprocs": [ + { + "func": "_shard_dense_h_to_4h_q_scale", + "out_shape": [2, 5120, 80], + "out_dtype": "float16", + } + ], + }, + { + "name": "gpt_neox.layers.6.mlp.dense_h_to_4h.bias", + "shape": [5120], + "dtype": "float32", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.6.mlp.dense_4h_to_h.q_weight", + "shape": [2560, 640], + "dtype": "uint32", + "preprocs": [ + { + "func_name": "_shard_dense_4h_to_h_q_weight", + "out_shape": [2, 2560, 640], + "out_dtype": "uint32", + } + ], + }, + { + "name": "gpt_neox.layers.6.mlp.dense_4h_to_h.q_scale", + "shape": [2560, 160], + "dtype": "float16", + "preprocs": [ + { + "func_name": "_shard_dense_4h_to_h_q_scale", + "out_shape": [2, 2560, 160], + "out_dtype": "float16", + } + ], + }, + { + "name": "gpt_neox.layers.6.mlp.dense_4h_to_h.bias", + "shape": [2560], + "dtype": "float32", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.7.input_layernorm.weight", + "shape": [2560], + "dtype": "float16", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.7.input_layernorm.bias", + "shape": [2560], + "dtype": "float16", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.7.post_attention_layernorm.weight", + "shape": [2560], + "dtype": "float16", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.7.post_attention_layernorm.bias", + "shape": [2560], + "dtype": "float16", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.7.attention.query_key_value.q_weight", + "shape": [3840, 320], + "dtype": "uint32", + "preprocs": [ + { + "func_name": "_shard_qkv_q_weight", + "out_shape": [2, 3840, 320], + "out_dtype": "uint32", + } + ], + }, + { + "name": "gpt_neox.layers.7.attention.query_key_value.q_scale", + "shape": [3840, 80], + "dtype": "float16", + "preprocs": [ + { + "func_name": "_shard_qkv_q_scale", + "out_shape": [2, 3840, 80], + "out_dtype": "float16", + } + ], + }, + { + "name": "gpt_neox.layers.7.attention.query_key_value.bias", + "shape": [3840], + "dtype": "float16", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.7.attention.dense.q_weight", + "shape": [2560, 160], + "dtype": "uint32", + "preprocs": [ + { + "func_name": "_shard_dense_q_weight", + "out_shape": [2, 2560, 160], + "out_dtype": "uint32", + } + ], + }, + { + "name": "gpt_neox.layers.7.attention.dense.q_scale", + "shape": [2560, 40], + "dtype": "float16", + "preprocs": [ + { + "func_name": "_shard_dense_q_scale", + "out_shape": [2, 2560, 40], + "out_dtype": "float16", + } + ], + }, + { + "name": "gpt_neox.layers.7.attention.dense.bias", + "shape": [2560], + "dtype": "float16", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.7.mlp.dense_h_to_4h.q_weight", + "shape": [5120, 320], + "dtype": "uint32", + "preprocs": [ + { + "func": "_shard_dense_h_to_4h_q_weight", + "out_shape": [2, 5120, 320], + "out_dtype": "uint32", + } + ], + }, + { + "name": "gpt_neox.layers.7.mlp.dense_h_to_4h.q_scale", + "shape": [5120, 80], + "dtype": "float16", + "preprocs": [ + { + "func": "_shard_dense_h_to_4h_q_scale", + "out_shape": [2, 5120, 80], + "out_dtype": "float16", + } + ], + }, + { + "name": "gpt_neox.layers.7.mlp.dense_h_to_4h.bias", + "shape": [5120], + "dtype": "float32", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.7.mlp.dense_4h_to_h.q_weight", + "shape": [2560, 640], + "dtype": "uint32", + "preprocs": [ + { + "func_name": "_shard_dense_4h_to_h_q_weight", + "out_shape": [2, 2560, 640], + "out_dtype": "uint32", + } + ], + }, + { + "name": "gpt_neox.layers.7.mlp.dense_4h_to_h.q_scale", + "shape": [2560, 160], + "dtype": "float16", + "preprocs": [ + { + "func_name": "_shard_dense_4h_to_h_q_scale", + "out_shape": [2, 2560, 160], + "out_dtype": "float16", + } + ], + }, + { + "name": "gpt_neox.layers.7.mlp.dense_4h_to_h.bias", + "shape": [2560], + "dtype": "float32", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.8.input_layernorm.weight", + "shape": [2560], + "dtype": "float16", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.8.input_layernorm.bias", + "shape": [2560], + "dtype": "float16", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.8.post_attention_layernorm.weight", + "shape": [2560], + "dtype": "float16", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.8.post_attention_layernorm.bias", + "shape": [2560], + "dtype": "float16", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.8.attention.query_key_value.q_weight", + "shape": [3840, 320], + "dtype": "uint32", + "preprocs": [ + { + "func_name": "_shard_qkv_q_weight", + "out_shape": [2, 3840, 320], + "out_dtype": "uint32", + } + ], + }, + { + "name": "gpt_neox.layers.8.attention.query_key_value.q_scale", + "shape": [3840, 80], + "dtype": "float16", + "preprocs": [ + { + "func_name": "_shard_qkv_q_scale", + "out_shape": [2, 3840, 80], + "out_dtype": "float16", + } + ], + }, + { + "name": "gpt_neox.layers.8.attention.query_key_value.bias", + "shape": [3840], + "dtype": "float16", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.8.attention.dense.q_weight", + "shape": [2560, 160], + "dtype": "uint32", + "preprocs": [ + { + "func_name": "_shard_dense_q_weight", + "out_shape": [2, 2560, 160], + "out_dtype": "uint32", + } + ], + }, + { + "name": "gpt_neox.layers.8.attention.dense.q_scale", + "shape": [2560, 40], + "dtype": "float16", + "preprocs": [ + { + "func_name": "_shard_dense_q_scale", + "out_shape": [2, 2560, 40], + "out_dtype": "float16", + } + ], + }, + { + "name": "gpt_neox.layers.8.attention.dense.bias", + "shape": [2560], + "dtype": "float16", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.8.mlp.dense_h_to_4h.q_weight", + "shape": [5120, 320], + "dtype": "uint32", + "preprocs": [ + { + "func": "_shard_dense_h_to_4h_q_weight", + "out_shape": [2, 5120, 320], + "out_dtype": "uint32", + } + ], + }, + { + "name": "gpt_neox.layers.8.mlp.dense_h_to_4h.q_scale", + "shape": [5120, 80], + "dtype": "float16", + "preprocs": [ + { + "func": "_shard_dense_h_to_4h_q_scale", + "out_shape": [2, 5120, 80], + "out_dtype": "float16", + } + ], + }, + { + "name": "gpt_neox.layers.8.mlp.dense_h_to_4h.bias", + "shape": [5120], + "dtype": "float32", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.8.mlp.dense_4h_to_h.q_weight", + "shape": [2560, 640], + "dtype": "uint32", + "preprocs": [ + { + "func_name": "_shard_dense_4h_to_h_q_weight", + "out_shape": [2, 2560, 640], + "out_dtype": "uint32", + } + ], + }, + { + "name": "gpt_neox.layers.8.mlp.dense_4h_to_h.q_scale", + "shape": [2560, 160], + "dtype": "float16", + "preprocs": [ + { + "func_name": "_shard_dense_4h_to_h_q_scale", + "out_shape": [2, 2560, 160], + "out_dtype": "float16", + } + ], + }, + { + "name": "gpt_neox.layers.8.mlp.dense_4h_to_h.bias", + "shape": [2560], + "dtype": "float32", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.9.input_layernorm.weight", + "shape": [2560], + "dtype": "float16", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.9.input_layernorm.bias", + "shape": [2560], + "dtype": "float16", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.9.post_attention_layernorm.weight", + "shape": [2560], + "dtype": "float16", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.9.post_attention_layernorm.bias", + "shape": [2560], + "dtype": "float16", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.9.attention.query_key_value.q_weight", + "shape": [3840, 320], + "dtype": "uint32", + "preprocs": [ + { + "func_name": "_shard_qkv_q_weight", + "out_shape": [2, 3840, 320], + "out_dtype": "uint32", + } + ], + }, + { + "name": "gpt_neox.layers.9.attention.query_key_value.q_scale", + "shape": [3840, 80], + "dtype": "float16", + "preprocs": [ + { + "func_name": "_shard_qkv_q_scale", + "out_shape": [2, 3840, 80], + "out_dtype": "float16", + } + ], + }, + { + "name": "gpt_neox.layers.9.attention.query_key_value.bias", + "shape": [3840], + "dtype": "float16", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.9.attention.dense.q_weight", + "shape": [2560, 160], + "dtype": "uint32", + "preprocs": [ + { + "func_name": "_shard_dense_q_weight", + "out_shape": [2, 2560, 160], + "out_dtype": "uint32", + } + ], + }, + { + "name": "gpt_neox.layers.9.attention.dense.q_scale", + "shape": [2560, 40], + "dtype": "float16", + "preprocs": [ + { + "func_name": "_shard_dense_q_scale", + "out_shape": [2, 2560, 40], + "out_dtype": "float16", + } + ], + }, + { + "name": "gpt_neox.layers.9.attention.dense.bias", + "shape": [2560], + "dtype": "float16", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.9.mlp.dense_h_to_4h.q_weight", + "shape": [5120, 320], + "dtype": "uint32", + "preprocs": [ + { + "func": "_shard_dense_h_to_4h_q_weight", + "out_shape": [2, 5120, 320], + "out_dtype": "uint32", + } + ], + }, + { + "name": "gpt_neox.layers.9.mlp.dense_h_to_4h.q_scale", + "shape": [5120, 80], + "dtype": "float16", + "preprocs": [ + { + "func": "_shard_dense_h_to_4h_q_scale", + "out_shape": [2, 5120, 80], + "out_dtype": "float16", + } + ], + }, + { + "name": "gpt_neox.layers.9.mlp.dense_h_to_4h.bias", + "shape": [5120], + "dtype": "float32", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.9.mlp.dense_4h_to_h.q_weight", + "shape": [2560, 640], + "dtype": "uint32", + "preprocs": [ + { + "func_name": "_shard_dense_4h_to_h_q_weight", + "out_shape": [2, 2560, 640], + "out_dtype": "uint32", + } + ], + }, + { + "name": "gpt_neox.layers.9.mlp.dense_4h_to_h.q_scale", + "shape": [2560, 160], + "dtype": "float16", + "preprocs": [ + { + "func_name": "_shard_dense_4h_to_h_q_scale", + "out_shape": [2, 2560, 160], + "out_dtype": "float16", + } + ], + }, + { + "name": "gpt_neox.layers.9.mlp.dense_4h_to_h.bias", + "shape": [2560], + "dtype": "float32", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.10.input_layernorm.weight", + "shape": [2560], + "dtype": "float16", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.10.input_layernorm.bias", + "shape": [2560], + "dtype": "float16", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.10.post_attention_layernorm.weight", + "shape": [2560], + "dtype": "float16", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.10.post_attention_layernorm.bias", + "shape": [2560], + "dtype": "float16", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.10.attention.query_key_value.q_weight", + "shape": [3840, 320], + "dtype": "uint32", + "preprocs": [ + { + "func_name": "_shard_qkv_q_weight", + "out_shape": [2, 3840, 320], + "out_dtype": "uint32", + } + ], + }, + { + "name": "gpt_neox.layers.10.attention.query_key_value.q_scale", + "shape": [3840, 80], + "dtype": "float16", + "preprocs": [ + { + "func_name": "_shard_qkv_q_scale", + "out_shape": [2, 3840, 80], + "out_dtype": "float16", + } + ], + }, + { + "name": "gpt_neox.layers.10.attention.query_key_value.bias", + "shape": [3840], + "dtype": "float16", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.10.attention.dense.q_weight", + "shape": [2560, 160], + "dtype": "uint32", + "preprocs": [ + { + "func_name": "_shard_dense_q_weight", + "out_shape": [2, 2560, 160], + "out_dtype": "uint32", + } + ], + }, + { + "name": "gpt_neox.layers.10.attention.dense.q_scale", + "shape": [2560, 40], + "dtype": "float16", + "preprocs": [ + { + "func_name": "_shard_dense_q_scale", + "out_shape": [2, 2560, 40], + "out_dtype": "float16", + } + ], + }, + { + "name": "gpt_neox.layers.10.attention.dense.bias", + "shape": [2560], + "dtype": "float16", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.10.mlp.dense_h_to_4h.q_weight", + "shape": [5120, 320], + "dtype": "uint32", + "preprocs": [ + { + "func": "_shard_dense_h_to_4h_q_weight", + "out_shape": [2, 5120, 320], + "out_dtype": "uint32", + } + ], + }, + { + "name": "gpt_neox.layers.10.mlp.dense_h_to_4h.q_scale", + "shape": [5120, 80], + "dtype": "float16", + "preprocs": [ + { + "func": "_shard_dense_h_to_4h_q_scale", + "out_shape": [2, 5120, 80], + "out_dtype": "float16", + } + ], + }, + { + "name": "gpt_neox.layers.10.mlp.dense_h_to_4h.bias", + "shape": [5120], + "dtype": "float32", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.10.mlp.dense_4h_to_h.q_weight", + "shape": [2560, 640], + "dtype": "uint32", + "preprocs": [ + { + "func_name": "_shard_dense_4h_to_h_q_weight", + "out_shape": [2, 2560, 640], + "out_dtype": "uint32", + } + ], + }, + { + "name": "gpt_neox.layers.10.mlp.dense_4h_to_h.q_scale", + "shape": [2560, 160], + "dtype": "float16", + "preprocs": [ + { + "func_name": "_shard_dense_4h_to_h_q_scale", + "out_shape": [2, 2560, 160], + "out_dtype": "float16", + } + ], + }, + { + "name": "gpt_neox.layers.10.mlp.dense_4h_to_h.bias", + "shape": [2560], + "dtype": "float32", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.11.input_layernorm.weight", + "shape": [2560], + "dtype": "float16", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.11.input_layernorm.bias", + "shape": [2560], + "dtype": "float16", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.11.post_attention_layernorm.weight", + "shape": [2560], + "dtype": "float16", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.11.post_attention_layernorm.bias", + "shape": [2560], + "dtype": "float16", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.11.attention.query_key_value.q_weight", + "shape": [3840, 320], + "dtype": "uint32", + "preprocs": [ + { + "func_name": "_shard_qkv_q_weight", + "out_shape": [2, 3840, 320], + "out_dtype": "uint32", + } + ], + }, + { + "name": "gpt_neox.layers.11.attention.query_key_value.q_scale", + "shape": [3840, 80], + "dtype": "float16", + "preprocs": [ + { + "func_name": "_shard_qkv_q_scale", + "out_shape": [2, 3840, 80], + "out_dtype": "float16", + } + ], + }, + { + "name": "gpt_neox.layers.11.attention.query_key_value.bias", + "shape": [3840], + "dtype": "float16", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.11.attention.dense.q_weight", + "shape": [2560, 160], + "dtype": "uint32", + "preprocs": [ + { + "func_name": "_shard_dense_q_weight", + "out_shape": [2, 2560, 160], + "out_dtype": "uint32", + } + ], + }, + { + "name": "gpt_neox.layers.11.attention.dense.q_scale", + "shape": [2560, 40], + "dtype": "float16", + "preprocs": [ + { + "func_name": "_shard_dense_q_scale", + "out_shape": [2, 2560, 40], + "out_dtype": "float16", + } + ], + }, + { + "name": "gpt_neox.layers.11.attention.dense.bias", + "shape": [2560], + "dtype": "float16", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.11.mlp.dense_h_to_4h.q_weight", + "shape": [5120, 320], + "dtype": "uint32", + "preprocs": [ + { + "func": "_shard_dense_h_to_4h_q_weight", + "out_shape": [2, 5120, 320], + "out_dtype": "uint32", + } + ], + }, + { + "name": "gpt_neox.layers.11.mlp.dense_h_to_4h.q_scale", + "shape": [5120, 80], + "dtype": "float16", + "preprocs": [ + { + "func": "_shard_dense_h_to_4h_q_scale", + "out_shape": [2, 5120, 80], + "out_dtype": "float16", + } + ], + }, + { + "name": "gpt_neox.layers.11.mlp.dense_h_to_4h.bias", + "shape": [5120], + "dtype": "float32", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.11.mlp.dense_4h_to_h.q_weight", + "shape": [2560, 640], + "dtype": "uint32", + "preprocs": [ + { + "func_name": "_shard_dense_4h_to_h_q_weight", + "out_shape": [2, 2560, 640], + "out_dtype": "uint32", + } + ], + }, + { + "name": "gpt_neox.layers.11.mlp.dense_4h_to_h.q_scale", + "shape": [2560, 160], + "dtype": "float16", + "preprocs": [ + { + "func_name": "_shard_dense_4h_to_h_q_scale", + "out_shape": [2, 2560, 160], + "out_dtype": "float16", + } + ], + }, + { + "name": "gpt_neox.layers.11.mlp.dense_4h_to_h.bias", + "shape": [2560], + "dtype": "float32", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.12.input_layernorm.weight", + "shape": [2560], + "dtype": "float16", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.12.input_layernorm.bias", + "shape": [2560], + "dtype": "float16", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.12.post_attention_layernorm.weight", + "shape": [2560], + "dtype": "float16", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.12.post_attention_layernorm.bias", + "shape": [2560], + "dtype": "float16", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.12.attention.query_key_value.q_weight", + "shape": [3840, 320], + "dtype": "uint32", + "preprocs": [ + { + "func_name": "_shard_qkv_q_weight", + "out_shape": [2, 3840, 320], + "out_dtype": "uint32", + } + ], + }, + { + "name": "gpt_neox.layers.12.attention.query_key_value.q_scale", + "shape": [3840, 80], + "dtype": "float16", + "preprocs": [ + { + "func_name": "_shard_qkv_q_scale", + "out_shape": [2, 3840, 80], + "out_dtype": "float16", + } + ], + }, + { + "name": "gpt_neox.layers.12.attention.query_key_value.bias", + "shape": [3840], + "dtype": "float16", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.12.attention.dense.q_weight", + "shape": [2560, 160], + "dtype": "uint32", + "preprocs": [ + { + "func_name": "_shard_dense_q_weight", + "out_shape": [2, 2560, 160], + "out_dtype": "uint32", + } + ], + }, + { + "name": "gpt_neox.layers.12.attention.dense.q_scale", + "shape": [2560, 40], + "dtype": "float16", + "preprocs": [ + { + "func_name": "_shard_dense_q_scale", + "out_shape": [2, 2560, 40], + "out_dtype": "float16", + } + ], + }, + { + "name": "gpt_neox.layers.12.attention.dense.bias", + "shape": [2560], + "dtype": "float16", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.12.mlp.dense_h_to_4h.q_weight", + "shape": [5120, 320], + "dtype": "uint32", + "preprocs": [ + { + "func": "_shard_dense_h_to_4h_q_weight", + "out_shape": [2, 5120, 320], + "out_dtype": "uint32", + } + ], + }, + { + "name": "gpt_neox.layers.12.mlp.dense_h_to_4h.q_scale", + "shape": [5120, 80], + "dtype": "float16", + "preprocs": [ + { + "func": "_shard_dense_h_to_4h_q_scale", + "out_shape": [2, 5120, 80], + "out_dtype": "float16", + } + ], + }, + { + "name": "gpt_neox.layers.12.mlp.dense_h_to_4h.bias", + "shape": [5120], + "dtype": "float32", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.12.mlp.dense_4h_to_h.q_weight", + "shape": [2560, 640], + "dtype": "uint32", + "preprocs": [ + { + "func_name": "_shard_dense_4h_to_h_q_weight", + "out_shape": [2, 2560, 640], + "out_dtype": "uint32", + } + ], + }, + { + "name": "gpt_neox.layers.12.mlp.dense_4h_to_h.q_scale", + "shape": [2560, 160], + "dtype": "float16", + "preprocs": [ + { + "func_name": "_shard_dense_4h_to_h_q_scale", + "out_shape": [2, 2560, 160], + "out_dtype": "float16", + } + ], + }, + { + "name": "gpt_neox.layers.12.mlp.dense_4h_to_h.bias", + "shape": [2560], + "dtype": "float32", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.13.input_layernorm.weight", + "shape": [2560], + "dtype": "float16", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.13.input_layernorm.bias", + "shape": [2560], + "dtype": "float16", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.13.post_attention_layernorm.weight", + "shape": [2560], + "dtype": "float16", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.13.post_attention_layernorm.bias", + "shape": [2560], + "dtype": "float16", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.13.attention.query_key_value.q_weight", + "shape": [3840, 320], + "dtype": "uint32", + "preprocs": [ + { + "func_name": "_shard_qkv_q_weight", + "out_shape": [2, 3840, 320], + "out_dtype": "uint32", + } + ], + }, + { + "name": "gpt_neox.layers.13.attention.query_key_value.q_scale", + "shape": [3840, 80], + "dtype": "float16", + "preprocs": [ + { + "func_name": "_shard_qkv_q_scale", + "out_shape": [2, 3840, 80], + "out_dtype": "float16", + } + ], + }, + { + "name": "gpt_neox.layers.13.attention.query_key_value.bias", + "shape": [3840], + "dtype": "float16", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.13.attention.dense.q_weight", + "shape": [2560, 160], + "dtype": "uint32", + "preprocs": [ + { + "func_name": "_shard_dense_q_weight", + "out_shape": [2, 2560, 160], + "out_dtype": "uint32", + } + ], + }, + { + "name": "gpt_neox.layers.13.attention.dense.q_scale", + "shape": [2560, 40], + "dtype": "float16", + "preprocs": [ + { + "func_name": "_shard_dense_q_scale", + "out_shape": [2, 2560, 40], + "out_dtype": "float16", + } + ], + }, + { + "name": "gpt_neox.layers.13.attention.dense.bias", + "shape": [2560], + "dtype": "float16", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.13.mlp.dense_h_to_4h.q_weight", + "shape": [5120, 320], + "dtype": "uint32", + "preprocs": [ + { + "func": "_shard_dense_h_to_4h_q_weight", + "out_shape": [2, 5120, 320], + "out_dtype": "uint32", + } + ], + }, + { + "name": "gpt_neox.layers.13.mlp.dense_h_to_4h.q_scale", + "shape": [5120, 80], + "dtype": "float16", + "preprocs": [ + { + "func": "_shard_dense_h_to_4h_q_scale", + "out_shape": [2, 5120, 80], + "out_dtype": "float16", + } + ], + }, + { + "name": "gpt_neox.layers.13.mlp.dense_h_to_4h.bias", + "shape": [5120], + "dtype": "float32", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.13.mlp.dense_4h_to_h.q_weight", + "shape": [2560, 640], + "dtype": "uint32", + "preprocs": [ + { + "func_name": "_shard_dense_4h_to_h_q_weight", + "out_shape": [2, 2560, 640], + "out_dtype": "uint32", + } + ], + }, + { + "name": "gpt_neox.layers.13.mlp.dense_4h_to_h.q_scale", + "shape": [2560, 160], + "dtype": "float16", + "preprocs": [ + { + "func_name": "_shard_dense_4h_to_h_q_scale", + "out_shape": [2, 2560, 160], + "out_dtype": "float16", + } + ], + }, + { + "name": "gpt_neox.layers.13.mlp.dense_4h_to_h.bias", + "shape": [2560], + "dtype": "float32", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.14.input_layernorm.weight", + "shape": [2560], + "dtype": "float16", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.14.input_layernorm.bias", + "shape": [2560], + "dtype": "float16", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.14.post_attention_layernorm.weight", + "shape": [2560], + "dtype": "float16", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.14.post_attention_layernorm.bias", + "shape": [2560], + "dtype": "float16", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.14.attention.query_key_value.q_weight", + "shape": [3840, 320], + "dtype": "uint32", + "preprocs": [ + { + "func_name": "_shard_qkv_q_weight", + "out_shape": [2, 3840, 320], + "out_dtype": "uint32", + } + ], + }, + { + "name": "gpt_neox.layers.14.attention.query_key_value.q_scale", + "shape": [3840, 80], + "dtype": "float16", + "preprocs": [ + { + "func_name": "_shard_qkv_q_scale", + "out_shape": [2, 3840, 80], + "out_dtype": "float16", + } + ], + }, + { + "name": "gpt_neox.layers.14.attention.query_key_value.bias", + "shape": [3840], + "dtype": "float16", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.14.attention.dense.q_weight", + "shape": [2560, 160], + "dtype": "uint32", + "preprocs": [ + { + "func_name": "_shard_dense_q_weight", + "out_shape": [2, 2560, 160], + "out_dtype": "uint32", + } + ], + }, + { + "name": "gpt_neox.layers.14.attention.dense.q_scale", + "shape": [2560, 40], + "dtype": "float16", + "preprocs": [ + { + "func_name": "_shard_dense_q_scale", + "out_shape": [2, 2560, 40], + "out_dtype": "float16", + } + ], + }, + { + "name": "gpt_neox.layers.14.attention.dense.bias", + "shape": [2560], + "dtype": "float16", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.14.mlp.dense_h_to_4h.q_weight", + "shape": [5120, 320], + "dtype": "uint32", + "preprocs": [ + { + "func": "_shard_dense_h_to_4h_q_weight", + "out_shape": [2, 5120, 320], + "out_dtype": "uint32", + } + ], + }, + { + "name": "gpt_neox.layers.14.mlp.dense_h_to_4h.q_scale", + "shape": [5120, 80], + "dtype": "float16", + "preprocs": [ + { + "func": "_shard_dense_h_to_4h_q_scale", + "out_shape": [2, 5120, 80], + "out_dtype": "float16", + } + ], + }, + { + "name": "gpt_neox.layers.14.mlp.dense_h_to_4h.bias", + "shape": [5120], + "dtype": "float32", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.14.mlp.dense_4h_to_h.q_weight", + "shape": [2560, 640], + "dtype": "uint32", + "preprocs": [ + { + "func_name": "_shard_dense_4h_to_h_q_weight", + "out_shape": [2, 2560, 640], + "out_dtype": "uint32", + } + ], + }, + { + "name": "gpt_neox.layers.14.mlp.dense_4h_to_h.q_scale", + "shape": [2560, 160], + "dtype": "float16", + "preprocs": [ + { + "func_name": "_shard_dense_4h_to_h_q_scale", + "out_shape": [2, 2560, 160], + "out_dtype": "float16", + } + ], + }, + { + "name": "gpt_neox.layers.14.mlp.dense_4h_to_h.bias", + "shape": [2560], + "dtype": "float32", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.15.input_layernorm.weight", + "shape": [2560], + "dtype": "float16", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.15.input_layernorm.bias", + "shape": [2560], + "dtype": "float16", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.15.post_attention_layernorm.weight", + "shape": [2560], + "dtype": "float16", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.15.post_attention_layernorm.bias", + "shape": [2560], + "dtype": "float16", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.15.attention.query_key_value.q_weight", + "shape": [3840, 320], + "dtype": "uint32", + "preprocs": [ + { + "func_name": "_shard_qkv_q_weight", + "out_shape": [2, 3840, 320], + "out_dtype": "uint32", + } + ], + }, + { + "name": "gpt_neox.layers.15.attention.query_key_value.q_scale", + "shape": [3840, 80], + "dtype": "float16", + "preprocs": [ + { + "func_name": "_shard_qkv_q_scale", + "out_shape": [2, 3840, 80], + "out_dtype": "float16", + } + ], + }, + { + "name": "gpt_neox.layers.15.attention.query_key_value.bias", + "shape": [3840], + "dtype": "float16", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.15.attention.dense.q_weight", + "shape": [2560, 160], + "dtype": "uint32", + "preprocs": [ + { + "func_name": "_shard_dense_q_weight", + "out_shape": [2, 2560, 160], + "out_dtype": "uint32", + } + ], + }, + { + "name": "gpt_neox.layers.15.attention.dense.q_scale", + "shape": [2560, 40], + "dtype": "float16", + "preprocs": [ + { + "func_name": "_shard_dense_q_scale", + "out_shape": [2, 2560, 40], + "out_dtype": "float16", + } + ], + }, + { + "name": "gpt_neox.layers.15.attention.dense.bias", + "shape": [2560], + "dtype": "float16", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.15.mlp.dense_h_to_4h.q_weight", + "shape": [5120, 320], + "dtype": "uint32", + "preprocs": [ + { + "func": "_shard_dense_h_to_4h_q_weight", + "out_shape": [2, 5120, 320], + "out_dtype": "uint32", + } + ], + }, + { + "name": "gpt_neox.layers.15.mlp.dense_h_to_4h.q_scale", + "shape": [5120, 80], + "dtype": "float16", + "preprocs": [ + { + "func": "_shard_dense_h_to_4h_q_scale", + "out_shape": [2, 5120, 80], + "out_dtype": "float16", + } + ], + }, + { + "name": "gpt_neox.layers.15.mlp.dense_h_to_4h.bias", + "shape": [5120], + "dtype": "float32", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.15.mlp.dense_4h_to_h.q_weight", + "shape": [2560, 640], + "dtype": "uint32", + "preprocs": [ + { + "func_name": "_shard_dense_4h_to_h_q_weight", + "out_shape": [2, 2560, 640], + "out_dtype": "uint32", + } + ], + }, + { + "name": "gpt_neox.layers.15.mlp.dense_4h_to_h.q_scale", + "shape": [2560, 160], + "dtype": "float16", + "preprocs": [ + { + "func_name": "_shard_dense_4h_to_h_q_scale", + "out_shape": [2, 2560, 160], + "out_dtype": "float16", + } + ], + }, + { + "name": "gpt_neox.layers.15.mlp.dense_4h_to_h.bias", + "shape": [2560], + "dtype": "float32", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.16.input_layernorm.weight", + "shape": [2560], + "dtype": "float16", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.16.input_layernorm.bias", + "shape": [2560], + "dtype": "float16", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.16.post_attention_layernorm.weight", + "shape": [2560], + "dtype": "float16", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.16.post_attention_layernorm.bias", + "shape": [2560], + "dtype": "float16", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.16.attention.query_key_value.q_weight", + "shape": [3840, 320], + "dtype": "uint32", + "preprocs": [ + { + "func_name": "_shard_qkv_q_weight", + "out_shape": [2, 3840, 320], + "out_dtype": "uint32", + } + ], + }, + { + "name": "gpt_neox.layers.16.attention.query_key_value.q_scale", + "shape": [3840, 80], + "dtype": "float16", + "preprocs": [ + { + "func_name": "_shard_qkv_q_scale", + "out_shape": [2, 3840, 80], + "out_dtype": "float16", + } + ], + }, + { + "name": "gpt_neox.layers.16.attention.query_key_value.bias", + "shape": [3840], + "dtype": "float16", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.16.attention.dense.q_weight", + "shape": [2560, 160], + "dtype": "uint32", + "preprocs": [ + { + "func_name": "_shard_dense_q_weight", + "out_shape": [2, 2560, 160], + "out_dtype": "uint32", + } + ], + }, + { + "name": "gpt_neox.layers.16.attention.dense.q_scale", + "shape": [2560, 40], + "dtype": "float16", + "preprocs": [ + { + "func_name": "_shard_dense_q_scale", + "out_shape": [2, 2560, 40], + "out_dtype": "float16", + } + ], + }, + { + "name": "gpt_neox.layers.16.attention.dense.bias", + "shape": [2560], + "dtype": "float16", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.16.mlp.dense_h_to_4h.q_weight", + "shape": [5120, 320], + "dtype": "uint32", + "preprocs": [ + { + "func": "_shard_dense_h_to_4h_q_weight", + "out_shape": [2, 5120, 320], + "out_dtype": "uint32", + } + ], + }, + { + "name": "gpt_neox.layers.16.mlp.dense_h_to_4h.q_scale", + "shape": [5120, 80], + "dtype": "float16", + "preprocs": [ + { + "func": "_shard_dense_h_to_4h_q_scale", + "out_shape": [2, 5120, 80], + "out_dtype": "float16", + } + ], + }, + { + "name": "gpt_neox.layers.16.mlp.dense_h_to_4h.bias", + "shape": [5120], + "dtype": "float32", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.16.mlp.dense_4h_to_h.q_weight", + "shape": [2560, 640], + "dtype": "uint32", + "preprocs": [ + { + "func_name": "_shard_dense_4h_to_h_q_weight", + "out_shape": [2, 2560, 640], + "out_dtype": "uint32", + } + ], + }, + { + "name": "gpt_neox.layers.16.mlp.dense_4h_to_h.q_scale", + "shape": [2560, 160], + "dtype": "float16", + "preprocs": [ + { + "func_name": "_shard_dense_4h_to_h_q_scale", + "out_shape": [2, 2560, 160], + "out_dtype": "float16", + } + ], + }, + { + "name": "gpt_neox.layers.16.mlp.dense_4h_to_h.bias", + "shape": [2560], + "dtype": "float32", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.17.input_layernorm.weight", + "shape": [2560], + "dtype": "float16", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.17.input_layernorm.bias", + "shape": [2560], + "dtype": "float16", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.17.post_attention_layernorm.weight", + "shape": [2560], + "dtype": "float16", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.17.post_attention_layernorm.bias", + "shape": [2560], + "dtype": "float16", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.17.attention.query_key_value.q_weight", + "shape": [3840, 320], + "dtype": "uint32", + "preprocs": [ + { + "func_name": "_shard_qkv_q_weight", + "out_shape": [2, 3840, 320], + "out_dtype": "uint32", + } + ], + }, + { + "name": "gpt_neox.layers.17.attention.query_key_value.q_scale", + "shape": [3840, 80], + "dtype": "float16", + "preprocs": [ + { + "func_name": "_shard_qkv_q_scale", + "out_shape": [2, 3840, 80], + "out_dtype": "float16", + } + ], + }, + { + "name": "gpt_neox.layers.17.attention.query_key_value.bias", + "shape": [3840], + "dtype": "float16", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.17.attention.dense.q_weight", + "shape": [2560, 160], + "dtype": "uint32", + "preprocs": [ + { + "func_name": "_shard_dense_q_weight", + "out_shape": [2, 2560, 160], + "out_dtype": "uint32", + } + ], + }, + { + "name": "gpt_neox.layers.17.attention.dense.q_scale", + "shape": [2560, 40], + "dtype": "float16", + "preprocs": [ + { + "func_name": "_shard_dense_q_scale", + "out_shape": [2, 2560, 40], + "out_dtype": "float16", + } + ], + }, + { + "name": "gpt_neox.layers.17.attention.dense.bias", + "shape": [2560], + "dtype": "float16", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.17.mlp.dense_h_to_4h.q_weight", + "shape": [5120, 320], + "dtype": "uint32", + "preprocs": [ + { + "func": "_shard_dense_h_to_4h_q_weight", + "out_shape": [2, 5120, 320], + "out_dtype": "uint32", + } + ], + }, + { + "name": "gpt_neox.layers.17.mlp.dense_h_to_4h.q_scale", + "shape": [5120, 80], + "dtype": "float16", + "preprocs": [ + { + "func": "_shard_dense_h_to_4h_q_scale", + "out_shape": [2, 5120, 80], + "out_dtype": "float16", + } + ], + }, + { + "name": "gpt_neox.layers.17.mlp.dense_h_to_4h.bias", + "shape": [5120], + "dtype": "float32", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.17.mlp.dense_4h_to_h.q_weight", + "shape": [2560, 640], + "dtype": "uint32", + "preprocs": [ + { + "func_name": "_shard_dense_4h_to_h_q_weight", + "out_shape": [2, 2560, 640], + "out_dtype": "uint32", + } + ], + }, + { + "name": "gpt_neox.layers.17.mlp.dense_4h_to_h.q_scale", + "shape": [2560, 160], + "dtype": "float16", + "preprocs": [ + { + "func_name": "_shard_dense_4h_to_h_q_scale", + "out_shape": [2, 2560, 160], + "out_dtype": "float16", + } + ], + }, + { + "name": "gpt_neox.layers.17.mlp.dense_4h_to_h.bias", + "shape": [2560], + "dtype": "float32", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.18.input_layernorm.weight", + "shape": [2560], + "dtype": "float16", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.18.input_layernorm.bias", + "shape": [2560], + "dtype": "float16", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.18.post_attention_layernorm.weight", + "shape": [2560], + "dtype": "float16", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.18.post_attention_layernorm.bias", + "shape": [2560], + "dtype": "float16", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.18.attention.query_key_value.q_weight", + "shape": [3840, 320], + "dtype": "uint32", + "preprocs": [ + { + "func_name": "_shard_qkv_q_weight", + "out_shape": [2, 3840, 320], + "out_dtype": "uint32", + } + ], + }, + { + "name": "gpt_neox.layers.18.attention.query_key_value.q_scale", + "shape": [3840, 80], + "dtype": "float16", + "preprocs": [ + { + "func_name": "_shard_qkv_q_scale", + "out_shape": [2, 3840, 80], + "out_dtype": "float16", + } + ], + }, + { + "name": "gpt_neox.layers.18.attention.query_key_value.bias", + "shape": [3840], + "dtype": "float16", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.18.attention.dense.q_weight", + "shape": [2560, 160], + "dtype": "uint32", + "preprocs": [ + { + "func_name": "_shard_dense_q_weight", + "out_shape": [2, 2560, 160], + "out_dtype": "uint32", + } + ], + }, + { + "name": "gpt_neox.layers.18.attention.dense.q_scale", + "shape": [2560, 40], + "dtype": "float16", + "preprocs": [ + { + "func_name": "_shard_dense_q_scale", + "out_shape": [2, 2560, 40], + "out_dtype": "float16", + } + ], + }, + { + "name": "gpt_neox.layers.18.attention.dense.bias", + "shape": [2560], + "dtype": "float16", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.18.mlp.dense_h_to_4h.q_weight", + "shape": [5120, 320], + "dtype": "uint32", + "preprocs": [ + { + "func": "_shard_dense_h_to_4h_q_weight", + "out_shape": [2, 5120, 320], + "out_dtype": "uint32", + } + ], + }, + { + "name": "gpt_neox.layers.18.mlp.dense_h_to_4h.q_scale", + "shape": [5120, 80], + "dtype": "float16", + "preprocs": [ + { + "func": "_shard_dense_h_to_4h_q_scale", + "out_shape": [2, 5120, 80], + "out_dtype": "float16", + } + ], + }, + { + "name": "gpt_neox.layers.18.mlp.dense_h_to_4h.bias", + "shape": [5120], + "dtype": "float32", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.18.mlp.dense_4h_to_h.q_weight", + "shape": [2560, 640], + "dtype": "uint32", + "preprocs": [ + { + "func_name": "_shard_dense_4h_to_h_q_weight", + "out_shape": [2, 2560, 640], + "out_dtype": "uint32", + } + ], + }, + { + "name": "gpt_neox.layers.18.mlp.dense_4h_to_h.q_scale", + "shape": [2560, 160], + "dtype": "float16", + "preprocs": [ + { + "func_name": "_shard_dense_4h_to_h_q_scale", + "out_shape": [2, 2560, 160], + "out_dtype": "float16", + } + ], + }, + { + "name": "gpt_neox.layers.18.mlp.dense_4h_to_h.bias", + "shape": [2560], + "dtype": "float32", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.19.input_layernorm.weight", + "shape": [2560], + "dtype": "float16", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.19.input_layernorm.bias", + "shape": [2560], + "dtype": "float16", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.19.post_attention_layernorm.weight", + "shape": [2560], + "dtype": "float16", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.19.post_attention_layernorm.bias", + "shape": [2560], + "dtype": "float16", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.19.attention.query_key_value.q_weight", + "shape": [3840, 320], + "dtype": "uint32", + "preprocs": [ + { + "func_name": "_shard_qkv_q_weight", + "out_shape": [2, 3840, 320], + "out_dtype": "uint32", + } + ], + }, + { + "name": "gpt_neox.layers.19.attention.query_key_value.q_scale", + "shape": [3840, 80], + "dtype": "float16", + "preprocs": [ + { + "func_name": "_shard_qkv_q_scale", + "out_shape": [2, 3840, 80], + "out_dtype": "float16", + } + ], + }, + { + "name": "gpt_neox.layers.19.attention.query_key_value.bias", + "shape": [3840], + "dtype": "float16", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.19.attention.dense.q_weight", + "shape": [2560, 160], + "dtype": "uint32", + "preprocs": [ + { + "func_name": "_shard_dense_q_weight", + "out_shape": [2, 2560, 160], + "out_dtype": "uint32", + } + ], + }, + { + "name": "gpt_neox.layers.19.attention.dense.q_scale", + "shape": [2560, 40], + "dtype": "float16", + "preprocs": [ + { + "func_name": "_shard_dense_q_scale", + "out_shape": [2, 2560, 40], + "out_dtype": "float16", + } + ], + }, + { + "name": "gpt_neox.layers.19.attention.dense.bias", + "shape": [2560], + "dtype": "float16", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.19.mlp.dense_h_to_4h.q_weight", + "shape": [5120, 320], + "dtype": "uint32", + "preprocs": [ + { + "func": "_shard_dense_h_to_4h_q_weight", + "out_shape": [2, 5120, 320], + "out_dtype": "uint32", + } + ], + }, + { + "name": "gpt_neox.layers.19.mlp.dense_h_to_4h.q_scale", + "shape": [5120, 80], + "dtype": "float16", + "preprocs": [ + { + "func": "_shard_dense_h_to_4h_q_scale", + "out_shape": [2, 5120, 80], + "out_dtype": "float16", + } + ], + }, + { + "name": "gpt_neox.layers.19.mlp.dense_h_to_4h.bias", + "shape": [5120], + "dtype": "float32", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.19.mlp.dense_4h_to_h.q_weight", + "shape": [2560, 640], + "dtype": "uint32", + "preprocs": [ + { + "func_name": "_shard_dense_4h_to_h_q_weight", + "out_shape": [2, 2560, 640], + "out_dtype": "uint32", + } + ], + }, + { + "name": "gpt_neox.layers.19.mlp.dense_4h_to_h.q_scale", + "shape": [2560, 160], + "dtype": "float16", + "preprocs": [ + { + "func_name": "_shard_dense_4h_to_h_q_scale", + "out_shape": [2, 2560, 160], + "out_dtype": "float16", + } + ], + }, + { + "name": "gpt_neox.layers.19.mlp.dense_4h_to_h.bias", + "shape": [2560], + "dtype": "float32", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.20.input_layernorm.weight", + "shape": [2560], + "dtype": "float16", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.20.input_layernorm.bias", + "shape": [2560], + "dtype": "float16", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.20.post_attention_layernorm.weight", + "shape": [2560], + "dtype": "float16", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.20.post_attention_layernorm.bias", + "shape": [2560], + "dtype": "float16", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.20.attention.query_key_value.q_weight", + "shape": [3840, 320], + "dtype": "uint32", + "preprocs": [ + { + "func_name": "_shard_qkv_q_weight", + "out_shape": [2, 3840, 320], + "out_dtype": "uint32", + } + ], + }, + { + "name": "gpt_neox.layers.20.attention.query_key_value.q_scale", + "shape": [3840, 80], + "dtype": "float16", + "preprocs": [ + { + "func_name": "_shard_qkv_q_scale", + "out_shape": [2, 3840, 80], + "out_dtype": "float16", + } + ], + }, + { + "name": "gpt_neox.layers.20.attention.query_key_value.bias", + "shape": [3840], + "dtype": "float16", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.20.attention.dense.q_weight", + "shape": [2560, 160], + "dtype": "uint32", + "preprocs": [ + { + "func_name": "_shard_dense_q_weight", + "out_shape": [2, 2560, 160], + "out_dtype": "uint32", + } + ], + }, + { + "name": "gpt_neox.layers.20.attention.dense.q_scale", + "shape": [2560, 40], + "dtype": "float16", + "preprocs": [ + { + "func_name": "_shard_dense_q_scale", + "out_shape": [2, 2560, 40], + "out_dtype": "float16", + } + ], + }, + { + "name": "gpt_neox.layers.20.attention.dense.bias", + "shape": [2560], + "dtype": "float16", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.20.mlp.dense_h_to_4h.q_weight", + "shape": [5120, 320], + "dtype": "uint32", + "preprocs": [ + { + "func": "_shard_dense_h_to_4h_q_weight", + "out_shape": [2, 5120, 320], + "out_dtype": "uint32", + } + ], + }, + { + "name": "gpt_neox.layers.20.mlp.dense_h_to_4h.q_scale", + "shape": [5120, 80], + "dtype": "float16", + "preprocs": [ + { + "func": "_shard_dense_h_to_4h_q_scale", + "out_shape": [2, 5120, 80], + "out_dtype": "float16", + } + ], + }, + { + "name": "gpt_neox.layers.20.mlp.dense_h_to_4h.bias", + "shape": [5120], + "dtype": "float32", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.20.mlp.dense_4h_to_h.q_weight", + "shape": [2560, 640], + "dtype": "uint32", + "preprocs": [ + { + "func_name": "_shard_dense_4h_to_h_q_weight", + "out_shape": [2, 2560, 640], + "out_dtype": "uint32", + } + ], + }, + { + "name": "gpt_neox.layers.20.mlp.dense_4h_to_h.q_scale", + "shape": [2560, 160], + "dtype": "float16", + "preprocs": [ + { + "func_name": "_shard_dense_4h_to_h_q_scale", + "out_shape": [2, 2560, 160], + "out_dtype": "float16", + } + ], + }, + { + "name": "gpt_neox.layers.20.mlp.dense_4h_to_h.bias", + "shape": [2560], + "dtype": "float32", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.21.input_layernorm.weight", + "shape": [2560], + "dtype": "float16", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.21.input_layernorm.bias", + "shape": [2560], + "dtype": "float16", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.21.post_attention_layernorm.weight", + "shape": [2560], + "dtype": "float16", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.21.post_attention_layernorm.bias", + "shape": [2560], + "dtype": "float16", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.21.attention.query_key_value.q_weight", + "shape": [3840, 320], + "dtype": "uint32", + "preprocs": [ + { + "func_name": "_shard_qkv_q_weight", + "out_shape": [2, 3840, 320], + "out_dtype": "uint32", + } + ], + }, + { + "name": "gpt_neox.layers.21.attention.query_key_value.q_scale", + "shape": [3840, 80], + "dtype": "float16", + "preprocs": [ + { + "func_name": "_shard_qkv_q_scale", + "out_shape": [2, 3840, 80], + "out_dtype": "float16", + } + ], + }, + { + "name": "gpt_neox.layers.21.attention.query_key_value.bias", + "shape": [3840], + "dtype": "float16", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.21.attention.dense.q_weight", + "shape": [2560, 160], + "dtype": "uint32", + "preprocs": [ + { + "func_name": "_shard_dense_q_weight", + "out_shape": [2, 2560, 160], + "out_dtype": "uint32", + } + ], + }, + { + "name": "gpt_neox.layers.21.attention.dense.q_scale", + "shape": [2560, 40], + "dtype": "float16", + "preprocs": [ + { + "func_name": "_shard_dense_q_scale", + "out_shape": [2, 2560, 40], + "out_dtype": "float16", + } + ], + }, + { + "name": "gpt_neox.layers.21.attention.dense.bias", + "shape": [2560], + "dtype": "float16", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.21.mlp.dense_h_to_4h.q_weight", + "shape": [5120, 320], + "dtype": "uint32", + "preprocs": [ + { + "func": "_shard_dense_h_to_4h_q_weight", + "out_shape": [2, 5120, 320], + "out_dtype": "uint32", + } + ], + }, + { + "name": "gpt_neox.layers.21.mlp.dense_h_to_4h.q_scale", + "shape": [5120, 80], + "dtype": "float16", + "preprocs": [ + { + "func": "_shard_dense_h_to_4h_q_scale", + "out_shape": [2, 5120, 80], + "out_dtype": "float16", + } + ], + }, + { + "name": "gpt_neox.layers.21.mlp.dense_h_to_4h.bias", + "shape": [5120], + "dtype": "float32", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.21.mlp.dense_4h_to_h.q_weight", + "shape": [2560, 640], + "dtype": "uint32", + "preprocs": [ + { + "func_name": "_shard_dense_4h_to_h_q_weight", + "out_shape": [2, 2560, 640], + "out_dtype": "uint32", + } + ], + }, + { + "name": "gpt_neox.layers.21.mlp.dense_4h_to_h.q_scale", + "shape": [2560, 160], + "dtype": "float16", + "preprocs": [ + { + "func_name": "_shard_dense_4h_to_h_q_scale", + "out_shape": [2, 2560, 160], + "out_dtype": "float16", + } + ], + }, + { + "name": "gpt_neox.layers.21.mlp.dense_4h_to_h.bias", + "shape": [2560], + "dtype": "float32", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.22.input_layernorm.weight", + "shape": [2560], + "dtype": "float16", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.22.input_layernorm.bias", + "shape": [2560], + "dtype": "float16", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.22.post_attention_layernorm.weight", + "shape": [2560], + "dtype": "float16", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.22.post_attention_layernorm.bias", + "shape": [2560], + "dtype": "float16", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.22.attention.query_key_value.q_weight", + "shape": [3840, 320], + "dtype": "uint32", + "preprocs": [ + { + "func_name": "_shard_qkv_q_weight", + "out_shape": [2, 3840, 320], + "out_dtype": "uint32", + } + ], + }, + { + "name": "gpt_neox.layers.22.attention.query_key_value.q_scale", + "shape": [3840, 80], + "dtype": "float16", + "preprocs": [ + { + "func_name": "_shard_qkv_q_scale", + "out_shape": [2, 3840, 80], + "out_dtype": "float16", + } + ], + }, + { + "name": "gpt_neox.layers.22.attention.query_key_value.bias", + "shape": [3840], + "dtype": "float16", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.22.attention.dense.q_weight", + "shape": [2560, 160], + "dtype": "uint32", + "preprocs": [ + { + "func_name": "_shard_dense_q_weight", + "out_shape": [2, 2560, 160], + "out_dtype": "uint32", + } + ], + }, + { + "name": "gpt_neox.layers.22.attention.dense.q_scale", + "shape": [2560, 40], + "dtype": "float16", + "preprocs": [ + { + "func_name": "_shard_dense_q_scale", + "out_shape": [2, 2560, 40], + "out_dtype": "float16", + } + ], + }, + { + "name": "gpt_neox.layers.22.attention.dense.bias", + "shape": [2560], + "dtype": "float16", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.22.mlp.dense_h_to_4h.q_weight", + "shape": [5120, 320], + "dtype": "uint32", + "preprocs": [ + { + "func": "_shard_dense_h_to_4h_q_weight", + "out_shape": [2, 5120, 320], + "out_dtype": "uint32", + } + ], + }, + { + "name": "gpt_neox.layers.22.mlp.dense_h_to_4h.q_scale", + "shape": [5120, 80], + "dtype": "float16", + "preprocs": [ + { + "func": "_shard_dense_h_to_4h_q_scale", + "out_shape": [2, 5120, 80], + "out_dtype": "float16", + } + ], + }, + { + "name": "gpt_neox.layers.22.mlp.dense_h_to_4h.bias", + "shape": [5120], + "dtype": "float32", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.22.mlp.dense_4h_to_h.q_weight", + "shape": [2560, 640], + "dtype": "uint32", + "preprocs": [ + { + "func_name": "_shard_dense_4h_to_h_q_weight", + "out_shape": [2, 2560, 640], + "out_dtype": "uint32", + } + ], + }, + { + "name": "gpt_neox.layers.22.mlp.dense_4h_to_h.q_scale", + "shape": [2560, 160], + "dtype": "float16", + "preprocs": [ + { + "func_name": "_shard_dense_4h_to_h_q_scale", + "out_shape": [2, 2560, 160], + "out_dtype": "float16", + } + ], + }, + { + "name": "gpt_neox.layers.22.mlp.dense_4h_to_h.bias", + "shape": [2560], + "dtype": "float32", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.23.input_layernorm.weight", + "shape": [2560], + "dtype": "float16", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.23.input_layernorm.bias", + "shape": [2560], + "dtype": "float16", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.23.post_attention_layernorm.weight", + "shape": [2560], + "dtype": "float16", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.23.post_attention_layernorm.bias", + "shape": [2560], + "dtype": "float16", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.23.attention.query_key_value.q_weight", + "shape": [3840, 320], + "dtype": "uint32", + "preprocs": [ + { + "func_name": "_shard_qkv_q_weight", + "out_shape": [2, 3840, 320], + "out_dtype": "uint32", + } + ], + }, + { + "name": "gpt_neox.layers.23.attention.query_key_value.q_scale", + "shape": [3840, 80], + "dtype": "float16", + "preprocs": [ + { + "func_name": "_shard_qkv_q_scale", + "out_shape": [2, 3840, 80], + "out_dtype": "float16", + } + ], + }, + { + "name": "gpt_neox.layers.23.attention.query_key_value.bias", + "shape": [3840], + "dtype": "float16", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.23.attention.dense.q_weight", + "shape": [2560, 160], + "dtype": "uint32", + "preprocs": [ + { + "func_name": "_shard_dense_q_weight", + "out_shape": [2, 2560, 160], + "out_dtype": "uint32", + } + ], + }, + { + "name": "gpt_neox.layers.23.attention.dense.q_scale", + "shape": [2560, 40], + "dtype": "float16", + "preprocs": [ + { + "func_name": "_shard_dense_q_scale", + "out_shape": [2, 2560, 40], + "out_dtype": "float16", + } + ], + }, + { + "name": "gpt_neox.layers.23.attention.dense.bias", + "shape": [2560], + "dtype": "float16", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.23.mlp.dense_h_to_4h.q_weight", + "shape": [5120, 320], + "dtype": "uint32", + "preprocs": [ + { + "func": "_shard_dense_h_to_4h_q_weight", + "out_shape": [2, 5120, 320], + "out_dtype": "uint32", + } + ], + }, + { + "name": "gpt_neox.layers.23.mlp.dense_h_to_4h.q_scale", + "shape": [5120, 80], + "dtype": "float16", + "preprocs": [ + { + "func": "_shard_dense_h_to_4h_q_scale", + "out_shape": [2, 5120, 80], + "out_dtype": "float16", + } + ], + }, + { + "name": "gpt_neox.layers.23.mlp.dense_h_to_4h.bias", + "shape": [5120], + "dtype": "float32", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.23.mlp.dense_4h_to_h.q_weight", + "shape": [2560, 640], + "dtype": "uint32", + "preprocs": [ + { + "func_name": "_shard_dense_4h_to_h_q_weight", + "out_shape": [2, 2560, 640], + "out_dtype": "uint32", + } + ], + }, + { + "name": "gpt_neox.layers.23.mlp.dense_4h_to_h.q_scale", + "shape": [2560, 160], + "dtype": "float16", + "preprocs": [ + { + "func_name": "_shard_dense_4h_to_h_q_scale", + "out_shape": [2, 2560, 160], + "out_dtype": "float16", + } + ], + }, + { + "name": "gpt_neox.layers.23.mlp.dense_4h_to_h.bias", + "shape": [2560], + "dtype": "float32", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.24.input_layernorm.weight", + "shape": [2560], + "dtype": "float16", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.24.input_layernorm.bias", + "shape": [2560], + "dtype": "float16", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.24.post_attention_layernorm.weight", + "shape": [2560], + "dtype": "float16", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.24.post_attention_layernorm.bias", + "shape": [2560], + "dtype": "float16", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.24.attention.query_key_value.q_weight", + "shape": [3840, 320], + "dtype": "uint32", + "preprocs": [ + { + "func_name": "_shard_qkv_q_weight", + "out_shape": [2, 3840, 320], + "out_dtype": "uint32", + } + ], + }, + { + "name": "gpt_neox.layers.24.attention.query_key_value.q_scale", + "shape": [3840, 80], + "dtype": "float16", + "preprocs": [ + { + "func_name": "_shard_qkv_q_scale", + "out_shape": [2, 3840, 80], + "out_dtype": "float16", + } + ], + }, + { + "name": "gpt_neox.layers.24.attention.query_key_value.bias", + "shape": [3840], + "dtype": "float16", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.24.attention.dense.q_weight", + "shape": [2560, 160], + "dtype": "uint32", + "preprocs": [ + { + "func_name": "_shard_dense_q_weight", + "out_shape": [2, 2560, 160], + "out_dtype": "uint32", + } + ], + }, + { + "name": "gpt_neox.layers.24.attention.dense.q_scale", + "shape": [2560, 40], + "dtype": "float16", + "preprocs": [ + { + "func_name": "_shard_dense_q_scale", + "out_shape": [2, 2560, 40], + "out_dtype": "float16", + } + ], + }, + { + "name": "gpt_neox.layers.24.attention.dense.bias", + "shape": [2560], + "dtype": "float16", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.24.mlp.dense_h_to_4h.q_weight", + "shape": [5120, 320], + "dtype": "uint32", + "preprocs": [ + { + "func": "_shard_dense_h_to_4h_q_weight", + "out_shape": [2, 5120, 320], + "out_dtype": "uint32", + } + ], + }, + { + "name": "gpt_neox.layers.24.mlp.dense_h_to_4h.q_scale", + "shape": [5120, 80], + "dtype": "float16", + "preprocs": [ + { + "func": "_shard_dense_h_to_4h_q_scale", + "out_shape": [2, 5120, 80], + "out_dtype": "float16", + } + ], + }, + { + "name": "gpt_neox.layers.24.mlp.dense_h_to_4h.bias", + "shape": [5120], + "dtype": "float32", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.24.mlp.dense_4h_to_h.q_weight", + "shape": [2560, 640], + "dtype": "uint32", + "preprocs": [ + { + "func_name": "_shard_dense_4h_to_h_q_weight", + "out_shape": [2, 2560, 640], + "out_dtype": "uint32", + } + ], + }, + { + "name": "gpt_neox.layers.24.mlp.dense_4h_to_h.q_scale", + "shape": [2560, 160], + "dtype": "float16", + "preprocs": [ + { + "func_name": "_shard_dense_4h_to_h_q_scale", + "out_shape": [2, 2560, 160], + "out_dtype": "float16", + } + ], + }, + { + "name": "gpt_neox.layers.24.mlp.dense_4h_to_h.bias", + "shape": [2560], + "dtype": "float32", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.25.input_layernorm.weight", + "shape": [2560], + "dtype": "float16", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.25.input_layernorm.bias", + "shape": [2560], + "dtype": "float16", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.25.post_attention_layernorm.weight", + "shape": [2560], + "dtype": "float16", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.25.post_attention_layernorm.bias", + "shape": [2560], + "dtype": "float16", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.25.attention.query_key_value.q_weight", + "shape": [3840, 320], + "dtype": "uint32", + "preprocs": [ + { + "func_name": "_shard_qkv_q_weight", + "out_shape": [2, 3840, 320], + "out_dtype": "uint32", + } + ], + }, + { + "name": "gpt_neox.layers.25.attention.query_key_value.q_scale", + "shape": [3840, 80], + "dtype": "float16", + "preprocs": [ + { + "func_name": "_shard_qkv_q_scale", + "out_shape": [2, 3840, 80], + "out_dtype": "float16", + } + ], + }, + { + "name": "gpt_neox.layers.25.attention.query_key_value.bias", + "shape": [3840], + "dtype": "float16", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.25.attention.dense.q_weight", + "shape": [2560, 160], + "dtype": "uint32", + "preprocs": [ + { + "func_name": "_shard_dense_q_weight", + "out_shape": [2, 2560, 160], + "out_dtype": "uint32", + } + ], + }, + { + "name": "gpt_neox.layers.25.attention.dense.q_scale", + "shape": [2560, 40], + "dtype": "float16", + "preprocs": [ + { + "func_name": "_shard_dense_q_scale", + "out_shape": [2, 2560, 40], + "out_dtype": "float16", + } + ], + }, + { + "name": "gpt_neox.layers.25.attention.dense.bias", + "shape": [2560], + "dtype": "float16", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.25.mlp.dense_h_to_4h.q_weight", + "shape": [5120, 320], + "dtype": "uint32", + "preprocs": [ + { + "func": "_shard_dense_h_to_4h_q_weight", + "out_shape": [2, 5120, 320], + "out_dtype": "uint32", + } + ], + }, + { + "name": "gpt_neox.layers.25.mlp.dense_h_to_4h.q_scale", + "shape": [5120, 80], + "dtype": "float16", + "preprocs": [ + { + "func": "_shard_dense_h_to_4h_q_scale", + "out_shape": [2, 5120, 80], + "out_dtype": "float16", + } + ], + }, + { + "name": "gpt_neox.layers.25.mlp.dense_h_to_4h.bias", + "shape": [5120], + "dtype": "float32", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.25.mlp.dense_4h_to_h.q_weight", + "shape": [2560, 640], + "dtype": "uint32", + "preprocs": [ + { + "func_name": "_shard_dense_4h_to_h_q_weight", + "out_shape": [2, 2560, 640], + "out_dtype": "uint32", + } + ], + }, + { + "name": "gpt_neox.layers.25.mlp.dense_4h_to_h.q_scale", + "shape": [2560, 160], + "dtype": "float16", + "preprocs": [ + { + "func_name": "_shard_dense_4h_to_h_q_scale", + "out_shape": [2, 2560, 160], + "out_dtype": "float16", + } + ], + }, + { + "name": "gpt_neox.layers.25.mlp.dense_4h_to_h.bias", + "shape": [2560], + "dtype": "float32", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.26.input_layernorm.weight", + "shape": [2560], + "dtype": "float16", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.26.input_layernorm.bias", + "shape": [2560], + "dtype": "float16", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.26.post_attention_layernorm.weight", + "shape": [2560], + "dtype": "float16", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.26.post_attention_layernorm.bias", + "shape": [2560], + "dtype": "float16", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.26.attention.query_key_value.q_weight", + "shape": [3840, 320], + "dtype": "uint32", + "preprocs": [ + { + "func_name": "_shard_qkv_q_weight", + "out_shape": [2, 3840, 320], + "out_dtype": "uint32", + } + ], + }, + { + "name": "gpt_neox.layers.26.attention.query_key_value.q_scale", + "shape": [3840, 80], + "dtype": "float16", + "preprocs": [ + { + "func_name": "_shard_qkv_q_scale", + "out_shape": [2, 3840, 80], + "out_dtype": "float16", + } + ], + }, + { + "name": "gpt_neox.layers.26.attention.query_key_value.bias", + "shape": [3840], + "dtype": "float16", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.26.attention.dense.q_weight", + "shape": [2560, 160], + "dtype": "uint32", + "preprocs": [ + { + "func_name": "_shard_dense_q_weight", + "out_shape": [2, 2560, 160], + "out_dtype": "uint32", + } + ], + }, + { + "name": "gpt_neox.layers.26.attention.dense.q_scale", + "shape": [2560, 40], + "dtype": "float16", + "preprocs": [ + { + "func_name": "_shard_dense_q_scale", + "out_shape": [2, 2560, 40], + "out_dtype": "float16", + } + ], + }, + { + "name": "gpt_neox.layers.26.attention.dense.bias", + "shape": [2560], + "dtype": "float16", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.26.mlp.dense_h_to_4h.q_weight", + "shape": [5120, 320], + "dtype": "uint32", + "preprocs": [ + { + "func": "_shard_dense_h_to_4h_q_weight", + "out_shape": [2, 5120, 320], + "out_dtype": "uint32", + } + ], + }, + { + "name": "gpt_neox.layers.26.mlp.dense_h_to_4h.q_scale", + "shape": [5120, 80], + "dtype": "float16", + "preprocs": [ + { + "func": "_shard_dense_h_to_4h_q_scale", + "out_shape": [2, 5120, 80], + "out_dtype": "float16", + } + ], + }, + { + "name": "gpt_neox.layers.26.mlp.dense_h_to_4h.bias", + "shape": [5120], + "dtype": "float32", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.26.mlp.dense_4h_to_h.q_weight", + "shape": [2560, 640], + "dtype": "uint32", + "preprocs": [ + { + "func_name": "_shard_dense_4h_to_h_q_weight", + "out_shape": [2, 2560, 640], + "out_dtype": "uint32", + } + ], + }, + { + "name": "gpt_neox.layers.26.mlp.dense_4h_to_h.q_scale", + "shape": [2560, 160], + "dtype": "float16", + "preprocs": [ + { + "func_name": "_shard_dense_4h_to_h_q_scale", + "out_shape": [2, 2560, 160], + "out_dtype": "float16", + } + ], + }, + { + "name": "gpt_neox.layers.26.mlp.dense_4h_to_h.bias", + "shape": [2560], + "dtype": "float32", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.27.input_layernorm.weight", + "shape": [2560], + "dtype": "float16", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.27.input_layernorm.bias", + "shape": [2560], + "dtype": "float16", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.27.post_attention_layernorm.weight", + "shape": [2560], + "dtype": "float16", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.27.post_attention_layernorm.bias", + "shape": [2560], + "dtype": "float16", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.27.attention.query_key_value.q_weight", + "shape": [3840, 320], + "dtype": "uint32", + "preprocs": [ + { + "func_name": "_shard_qkv_q_weight", + "out_shape": [2, 3840, 320], + "out_dtype": "uint32", + } + ], + }, + { + "name": "gpt_neox.layers.27.attention.query_key_value.q_scale", + "shape": [3840, 80], + "dtype": "float16", + "preprocs": [ + { + "func_name": "_shard_qkv_q_scale", + "out_shape": [2, 3840, 80], + "out_dtype": "float16", + } + ], + }, + { + "name": "gpt_neox.layers.27.attention.query_key_value.bias", + "shape": [3840], + "dtype": "float16", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.27.attention.dense.q_weight", + "shape": [2560, 160], + "dtype": "uint32", + "preprocs": [ + { + "func_name": "_shard_dense_q_weight", + "out_shape": [2, 2560, 160], + "out_dtype": "uint32", + } + ], + }, + { + "name": "gpt_neox.layers.27.attention.dense.q_scale", + "shape": [2560, 40], + "dtype": "float16", + "preprocs": [ + { + "func_name": "_shard_dense_q_scale", + "out_shape": [2, 2560, 40], + "out_dtype": "float16", + } + ], + }, + { + "name": "gpt_neox.layers.27.attention.dense.bias", + "shape": [2560], + "dtype": "float16", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.27.mlp.dense_h_to_4h.q_weight", + "shape": [5120, 320], + "dtype": "uint32", + "preprocs": [ + { + "func": "_shard_dense_h_to_4h_q_weight", + "out_shape": [2, 5120, 320], + "out_dtype": "uint32", + } + ], + }, + { + "name": "gpt_neox.layers.27.mlp.dense_h_to_4h.q_scale", + "shape": [5120, 80], + "dtype": "float16", + "preprocs": [ + { + "func": "_shard_dense_h_to_4h_q_scale", + "out_shape": [2, 5120, 80], + "out_dtype": "float16", + } + ], + }, + { + "name": "gpt_neox.layers.27.mlp.dense_h_to_4h.bias", + "shape": [5120], + "dtype": "float32", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.27.mlp.dense_4h_to_h.q_weight", + "shape": [2560, 640], + "dtype": "uint32", + "preprocs": [ + { + "func_name": "_shard_dense_4h_to_h_q_weight", + "out_shape": [2, 2560, 640], + "out_dtype": "uint32", + } + ], + }, + { + "name": "gpt_neox.layers.27.mlp.dense_4h_to_h.q_scale", + "shape": [2560, 160], + "dtype": "float16", + "preprocs": [ + { + "func_name": "_shard_dense_4h_to_h_q_scale", + "out_shape": [2, 2560, 160], + "out_dtype": "float16", + } + ], + }, + { + "name": "gpt_neox.layers.27.mlp.dense_4h_to_h.bias", + "shape": [2560], + "dtype": "float32", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.28.input_layernorm.weight", + "shape": [2560], + "dtype": "float16", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.28.input_layernorm.bias", + "shape": [2560], + "dtype": "float16", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.28.post_attention_layernorm.weight", + "shape": [2560], + "dtype": "float16", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.28.post_attention_layernorm.bias", + "shape": [2560], + "dtype": "float16", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.28.attention.query_key_value.q_weight", + "shape": [3840, 320], + "dtype": "uint32", + "preprocs": [ + { + "func_name": "_shard_qkv_q_weight", + "out_shape": [2, 3840, 320], + "out_dtype": "uint32", + } + ], + }, + { + "name": "gpt_neox.layers.28.attention.query_key_value.q_scale", + "shape": [3840, 80], + "dtype": "float16", + "preprocs": [ + { + "func_name": "_shard_qkv_q_scale", + "out_shape": [2, 3840, 80], + "out_dtype": "float16", + } + ], + }, + { + "name": "gpt_neox.layers.28.attention.query_key_value.bias", + "shape": [3840], + "dtype": "float16", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.28.attention.dense.q_weight", + "shape": [2560, 160], + "dtype": "uint32", + "preprocs": [ + { + "func_name": "_shard_dense_q_weight", + "out_shape": [2, 2560, 160], + "out_dtype": "uint32", + } + ], + }, + { + "name": "gpt_neox.layers.28.attention.dense.q_scale", + "shape": [2560, 40], + "dtype": "float16", + "preprocs": [ + { + "func_name": "_shard_dense_q_scale", + "out_shape": [2, 2560, 40], + "out_dtype": "float16", + } + ], + }, + { + "name": "gpt_neox.layers.28.attention.dense.bias", + "shape": [2560], + "dtype": "float16", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.28.mlp.dense_h_to_4h.q_weight", + "shape": [5120, 320], + "dtype": "uint32", + "preprocs": [ + { + "func": "_shard_dense_h_to_4h_q_weight", + "out_shape": [2, 5120, 320], + "out_dtype": "uint32", + } + ], + }, + { + "name": "gpt_neox.layers.28.mlp.dense_h_to_4h.q_scale", + "shape": [5120, 80], + "dtype": "float16", + "preprocs": [ + { + "func": "_shard_dense_h_to_4h_q_scale", + "out_shape": [2, 5120, 80], + "out_dtype": "float16", + } + ], + }, + { + "name": "gpt_neox.layers.28.mlp.dense_h_to_4h.bias", + "shape": [5120], + "dtype": "float32", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.28.mlp.dense_4h_to_h.q_weight", + "shape": [2560, 640], + "dtype": "uint32", + "preprocs": [ + { + "func_name": "_shard_dense_4h_to_h_q_weight", + "out_shape": [2, 2560, 640], + "out_dtype": "uint32", + } + ], + }, + { + "name": "gpt_neox.layers.28.mlp.dense_4h_to_h.q_scale", + "shape": [2560, 160], + "dtype": "float16", + "preprocs": [ + { + "func_name": "_shard_dense_4h_to_h_q_scale", + "out_shape": [2, 2560, 160], + "out_dtype": "float16", + } + ], + }, + { + "name": "gpt_neox.layers.28.mlp.dense_4h_to_h.bias", + "shape": [2560], + "dtype": "float32", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.29.input_layernorm.weight", + "shape": [2560], + "dtype": "float16", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.29.input_layernorm.bias", + "shape": [2560], + "dtype": "float16", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.29.post_attention_layernorm.weight", + "shape": [2560], + "dtype": "float16", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.29.post_attention_layernorm.bias", + "shape": [2560], + "dtype": "float16", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.29.attention.query_key_value.q_weight", + "shape": [3840, 320], + "dtype": "uint32", + "preprocs": [ + { + "func_name": "_shard_qkv_q_weight", + "out_shape": [2, 3840, 320], + "out_dtype": "uint32", + } + ], + }, + { + "name": "gpt_neox.layers.29.attention.query_key_value.q_scale", + "shape": [3840, 80], + "dtype": "float16", + "preprocs": [ + { + "func_name": "_shard_qkv_q_scale", + "out_shape": [2, 3840, 80], + "out_dtype": "float16", + } + ], + }, + { + "name": "gpt_neox.layers.29.attention.query_key_value.bias", + "shape": [3840], + "dtype": "float16", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.29.attention.dense.q_weight", + "shape": [2560, 160], + "dtype": "uint32", + "preprocs": [ + { + "func_name": "_shard_dense_q_weight", + "out_shape": [2, 2560, 160], + "out_dtype": "uint32", + } + ], + }, + { + "name": "gpt_neox.layers.29.attention.dense.q_scale", + "shape": [2560, 40], + "dtype": "float16", + "preprocs": [ + { + "func_name": "_shard_dense_q_scale", + "out_shape": [2, 2560, 40], + "out_dtype": "float16", + } + ], + }, + { + "name": "gpt_neox.layers.29.attention.dense.bias", + "shape": [2560], + "dtype": "float16", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.29.mlp.dense_h_to_4h.q_weight", + "shape": [5120, 320], + "dtype": "uint32", + "preprocs": [ + { + "func": "_shard_dense_h_to_4h_q_weight", + "out_shape": [2, 5120, 320], + "out_dtype": "uint32", + } + ], + }, + { + "name": "gpt_neox.layers.29.mlp.dense_h_to_4h.q_scale", + "shape": [5120, 80], + "dtype": "float16", + "preprocs": [ + { + "func": "_shard_dense_h_to_4h_q_scale", + "out_shape": [2, 5120, 80], + "out_dtype": "float16", + } + ], + }, + { + "name": "gpt_neox.layers.29.mlp.dense_h_to_4h.bias", + "shape": [5120], + "dtype": "float32", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.29.mlp.dense_4h_to_h.q_weight", + "shape": [2560, 640], + "dtype": "uint32", + "preprocs": [ + { + "func_name": "_shard_dense_4h_to_h_q_weight", + "out_shape": [2, 2560, 640], + "out_dtype": "uint32", + } + ], + }, + { + "name": "gpt_neox.layers.29.mlp.dense_4h_to_h.q_scale", + "shape": [2560, 160], + "dtype": "float16", + "preprocs": [ + { + "func_name": "_shard_dense_4h_to_h_q_scale", + "out_shape": [2, 2560, 160], + "out_dtype": "float16", + } + ], + }, + { + "name": "gpt_neox.layers.29.mlp.dense_4h_to_h.bias", + "shape": [2560], + "dtype": "float32", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.30.input_layernorm.weight", + "shape": [2560], + "dtype": "float16", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.30.input_layernorm.bias", + "shape": [2560], + "dtype": "float16", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.30.post_attention_layernorm.weight", + "shape": [2560], + "dtype": "float16", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.30.post_attention_layernorm.bias", + "shape": [2560], + "dtype": "float16", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.30.attention.query_key_value.q_weight", + "shape": [3840, 320], + "dtype": "uint32", + "preprocs": [ + { + "func_name": "_shard_qkv_q_weight", + "out_shape": [2, 3840, 320], + "out_dtype": "uint32", + } + ], + }, + { + "name": "gpt_neox.layers.30.attention.query_key_value.q_scale", + "shape": [3840, 80], + "dtype": "float16", + "preprocs": [ + { + "func_name": "_shard_qkv_q_scale", + "out_shape": [2, 3840, 80], + "out_dtype": "float16", + } + ], + }, + { + "name": "gpt_neox.layers.30.attention.query_key_value.bias", + "shape": [3840], + "dtype": "float16", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.30.attention.dense.q_weight", + "shape": [2560, 160], + "dtype": "uint32", + "preprocs": [ + { + "func_name": "_shard_dense_q_weight", + "out_shape": [2, 2560, 160], + "out_dtype": "uint32", + } + ], + }, + { + "name": "gpt_neox.layers.30.attention.dense.q_scale", + "shape": [2560, 40], + "dtype": "float16", + "preprocs": [ + { + "func_name": "_shard_dense_q_scale", + "out_shape": [2, 2560, 40], + "out_dtype": "float16", + } + ], + }, + { + "name": "gpt_neox.layers.30.attention.dense.bias", + "shape": [2560], + "dtype": "float16", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.30.mlp.dense_h_to_4h.q_weight", + "shape": [5120, 320], + "dtype": "uint32", + "preprocs": [ + { + "func": "_shard_dense_h_to_4h_q_weight", + "out_shape": [2, 5120, 320], + "out_dtype": "uint32", + } + ], + }, + { + "name": "gpt_neox.layers.30.mlp.dense_h_to_4h.q_scale", + "shape": [5120, 80], + "dtype": "float16", + "preprocs": [ + { + "func": "_shard_dense_h_to_4h_q_scale", + "out_shape": [2, 5120, 80], + "out_dtype": "float16", + } + ], + }, + { + "name": "gpt_neox.layers.30.mlp.dense_h_to_4h.bias", + "shape": [5120], + "dtype": "float32", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.30.mlp.dense_4h_to_h.q_weight", + "shape": [2560, 640], + "dtype": "uint32", + "preprocs": [ + { + "func_name": "_shard_dense_4h_to_h_q_weight", + "out_shape": [2, 2560, 640], + "out_dtype": "uint32", + } + ], + }, + { + "name": "gpt_neox.layers.30.mlp.dense_4h_to_h.q_scale", + "shape": [2560, 160], + "dtype": "float16", + "preprocs": [ + { + "func_name": "_shard_dense_4h_to_h_q_scale", + "out_shape": [2, 2560, 160], + "out_dtype": "float16", + } + ], + }, + { + "name": "gpt_neox.layers.30.mlp.dense_4h_to_h.bias", + "shape": [2560], + "dtype": "float32", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.31.input_layernorm.weight", + "shape": [2560], + "dtype": "float16", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.31.input_layernorm.bias", + "shape": [2560], + "dtype": "float16", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.31.post_attention_layernorm.weight", + "shape": [2560], + "dtype": "float16", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.31.post_attention_layernorm.bias", + "shape": [2560], + "dtype": "float16", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.31.attention.query_key_value.q_weight", + "shape": [3840, 320], + "dtype": "uint32", + "preprocs": [ + { + "func_name": "_shard_qkv_q_weight", + "out_shape": [2, 3840, 320], + "out_dtype": "uint32", + } + ], + }, + { + "name": "gpt_neox.layers.31.attention.query_key_value.q_scale", + "shape": [3840, 80], + "dtype": "float16", + "preprocs": [ + { + "func_name": "_shard_qkv_q_scale", + "out_shape": [2, 3840, 80], + "out_dtype": "float16", + } + ], + }, + { + "name": "gpt_neox.layers.31.attention.query_key_value.bias", + "shape": [3840], + "dtype": "float16", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.31.attention.dense.q_weight", + "shape": [2560, 160], + "dtype": "uint32", + "preprocs": [ + { + "func_name": "_shard_dense_q_weight", + "out_shape": [2, 2560, 160], + "out_dtype": "uint32", + } + ], + }, + { + "name": "gpt_neox.layers.31.attention.dense.q_scale", + "shape": [2560, 40], + "dtype": "float16", + "preprocs": [ + { + "func_name": "_shard_dense_q_scale", + "out_shape": [2, 2560, 40], + "out_dtype": "float16", + } + ], + }, + { + "name": "gpt_neox.layers.31.attention.dense.bias", + "shape": [2560], + "dtype": "float16", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.31.mlp.dense_h_to_4h.q_weight", + "shape": [5120, 320], + "dtype": "uint32", + "preprocs": [ + { + "func": "_shard_dense_h_to_4h_q_weight", + "out_shape": [2, 5120, 320], + "out_dtype": "uint32", + } + ], + }, + { + "name": "gpt_neox.layers.31.mlp.dense_h_to_4h.q_scale", + "shape": [5120, 80], + "dtype": "float16", + "preprocs": [ + { + "func": "_shard_dense_h_to_4h_q_scale", + "out_shape": [2, 5120, 80], + "out_dtype": "float16", + } + ], + }, + { + "name": "gpt_neox.layers.31.mlp.dense_h_to_4h.bias", + "shape": [5120], + "dtype": "float32", + "preprocs": [], + }, + { + "name": "gpt_neox.layers.31.mlp.dense_4h_to_h.q_weight", + "shape": [2560, 640], + "dtype": "uint32", + "preprocs": [ + { + "func_name": "_shard_dense_4h_to_h_q_weight", + "out_shape": [2, 2560, 640], + "out_dtype": "uint32", + } + ], + }, + { + "name": "gpt_neox.layers.31.mlp.dense_4h_to_h.q_scale", + "shape": [2560, 160], + "dtype": "float16", + "preprocs": [ + { + "func_name": "_shard_dense_4h_to_h_q_scale", + "out_shape": [2, 2560, 160], + "out_dtype": "float16", + } + ], + }, + { + "name": "gpt_neox.layers.31.mlp.dense_4h_to_h.bias", + "shape": [2560], + "dtype": "float32", + "preprocs": [], + }, + { + "name": "gpt_neox.final_layer_norm.weight", + "shape": [2560], + "dtype": "float16", + "preprocs": [], + }, + { + "name": "gpt_neox.final_layer_norm.bias", + "shape": [2560], + "dtype": "float16", + "preprocs": [], + }, + {"name": "embed_out.q_weight", "shape": [50432, 320], "dtype": "uint32", "preprocs": []}, + {"name": "embed_out.q_scale", "shape": [50432, 80], "dtype": "float16", "preprocs": []}, + ], + "memory_usage": { + "_initialize_effect": 0, + "decode": 16155140, + "prefill": 460539904, + "softmax_with_temperature": 0, + }, +} diff --git a/wasted/codetest1.py b/wasted/codetest1.py new file mode 100644 index 0000000000..6fcec18eba --- /dev/null +++ b/wasted/codetest1.py @@ -0,0 +1,24 @@ +# pylint: skip-file +cnt = 0 + + +def bt(nums, target, cur_i, cur_v): + global cnt + if cur_i == len(nums): + if cur_v == target: + cnt = cnt + 1 + else: + pass + else: + bt(nums, target, cur_i + 1, cur_v + nums[cur_i]) + bt(nums, target, cur_i + 1, cur_v - nums[cur_i]) + + +def f(nums, target): + bt(nums, target, cur_i=0, cur_v=0) + return cnt + + +nums = [1, 1, 1, 1, 1] +target = 3 +print(f(nums, target)) diff --git a/wasted/deploy_mlc.py b/wasted/deploy_mlc.py new file mode 100644 index 0000000000..3a3ac9fb36 --- /dev/null +++ b/wasted/deploy_mlc.py @@ -0,0 +1,102 @@ +import argparse +import os +import pickle + +import numpy as np +import torch +import tvm +from mlc_llm import utils +from transformers import AutoTokenizer +from tvm import relax + + +class TestState: + def __init__(self, args): + self.primary_device = tvm.device(args.primary_device) + ex = tvm.runtime.load_module( + os.path.join( + args.artifact_path, + f"{args.model}-{args.quantization.name}-{args.primary_device}.so", + ) + ) + self.vm = relax.VirtualMachine(ex, self.primary_device) + self.sess = None + + +def deploy_to_pipeline(args) -> None: + @tvm.register_func("debug_print") + def debug_print(dummy_object, info, a): + print(a.numpy()) + return dummy_object + + primary_device = tvm.device(args.primary_device) + const_params = utils.load_params(args.artifact_path, primary_device) + state = TestState(args) + tokenizer = AutoTokenizer.from_pretrained( + os.path.join(args.artifact_path, "params"), trust_remote_code=True + ) + + print("Tokenizing...") + inputs = tokenizer(args.prompt, return_tensors="pt").input_ids.to(torch.int32).numpy() + first_sampled_token = tvm.nd.array(np.array([[6234]]).astype("int32"), primary_device) + seq_len_shape = tvm.runtime.ShapeTuple([inputs.shape[1]]) + second_seq_len_shape = tvm.runtime.ShapeTuple([inputs.shape[1] + 1]) + dummy_obj = tvm.nd.array(np.array([[6234]]).astype("int32"), primary_device) + kv_caches = state.vm["create_kv_cache"]() + + print("Running inference...") + print("======================= Starts Encoding =======================") + + try: + prefill_func = state.vm["prefill"] + except AttributeError: + prefill_func = None + + if inputs.shape[1] > 1 and prefill_func: + inputs = tvm.nd.array(inputs, device=primary_device) + logits, kv_caches, dummy_obj = prefill_func( + inputs, seq_len_shape, kv_caches, dummy_obj, const_params + ) + else: + for i in range(inputs.shape[1]): + input_slice = tvm.nd.array(inputs[:, i : i + 1], device=primary_device) + logits, kv_caches = state.vm["decode"]( + input_slice, seq_len_shape, kv_caches, const_params + ) + + exit(0) + print("======================= Starts Decoding =======================") + logits, kv_caches, dummy_obj = state.vm["decode"]( + first_sampled_token, second_seq_len_shape, kv_caches, dummy_obj, const_params + ) + + +def _parse_args(): + args = argparse.ArgumentParser() + args.add_argument("--local-id", type=str, required=True) + args.add_argument("--artifact-path", type=str, default="dist") + args.add_argument("--primary-device", type=str, default="auto") + args.add_argument("--prompt", type=str, default="What is the meaning of life?") + args.add_argument("--time-eval", default=False, action="store_true") + args.add_argument("--skip-rounds", type=int, default=0) + parsed = args.parse_args() + parsed.model, parsed.quantization = parsed.local_id.rsplit("-", 1) + utils.argparse_postproc_common(parsed) + + parsed.artifact_path = os.path.join( + parsed.artifact_path, f"{parsed.model}-{parsed.quantization.name}" + ) + + if parsed.primary_device == "auto": + if tvm.cuda().exist: + parsed.primary_device = "cuda" + elif tvm.metal().exist: + parsed.primary_device = "metal" + else: + raise ValueError("Cannot auto deduce device-name, please set it") + return parsed + + +if __name__ == "__main__": + args = _parse_args() + deploy_to_pipeline(args) diff --git a/wasted/deploy_new_mlc.py b/wasted/deploy_new_mlc.py new file mode 100644 index 0000000000..adf101ca7e --- /dev/null +++ b/wasted/deploy_new_mlc.py @@ -0,0 +1,110 @@ +import argparse +import os +import pickle + +import numpy as np +import torch +import tvm +from mlc_llm import utils +from transformers import AutoTokenizer +from tvm import relax + + +def load_params(artifact_path: str, device, param_names): + from tvm.contrib import tvmjs # pylint: disable=import-outside-toplevel + + params, meta = tvmjs.load_ndarray_cache(f"{artifact_path}/params", device) + plist = [] + size = meta["ParamSize"] + for pname in param_names: + plist.append(params[pname]) + return plist + + +class TestState: + def __init__(self, args): + self.primary_device = tvm.device(args.primary_device) + ex = tvm.runtime.load_module(os.path.join(args.artifact_path, f"model.so")) + self.vm = relax.VirtualMachine(ex, self.primary_device) + self.sess = None + + +def deploy_to_pipeline(args) -> None: + state = TestState(args) + x = state.vm["_metadata"]() + metadata = eval(x) + params = metadata["params"] + param_names = [] + for param in params: + param_names.append(param["name"]) + + primary_device = tvm.device(args.primary_device) + const_params = load_params(args.artifact_path, primary_device, param_names) + tokenizer = AutoTokenizer.from_pretrained( + os.path.join(args.artifact_path, "params"), trust_remote_code=True + ) + + print("Tokenizing...") + inputs = tokenizer(args.prompt, return_tensors="pt").input_ids.to(torch.int32).numpy() + print(inputs) + first_sampled_token = tvm.nd.array(np.array([[6234]]).astype("int32"), primary_device) + seq_len_shape = tvm.runtime.ShapeTuple([inputs.shape[1]]) + second_seq_len_shape = tvm.runtime.ShapeTuple([inputs.shape[1] + 1]) + kv_caches = state.vm["_initialize_effect"]() + + print("Running inference...") + print("======================= Starts Encoding =======================") + + try: + prefill_func = state.vm["prefill"] + except AttributeError: + prefill_func = None + + if inputs.shape[1] > 1 and prefill_func: + inputs = tvm.nd.array(inputs, device=primary_device) + logits, kv_caches = prefill_func(inputs, seq_len_shape, kv_caches, const_params) + else: + for i in range(inputs.shape[1]): + input_slice = tvm.nd.array(inputs[:, i : i + 1], device=primary_device) + logits, kv_caches = state.vm["decode"]( + input_slice, seq_len_shape, kv_caches, const_params + ) + + exit(0) + print("======================= Starts Decoding =======================") + logits, kv_caches = state.vm["decode"]( + first_sampled_token, second_seq_len_shape, kv_caches, const_params + ) + + +def _parse_args(): + args = argparse.ArgumentParser() + args.add_argument("--local-id", type=str, default="") + args.add_argument("--artifact-path", type=str, default="dist") + args.add_argument("--primary-device", type=str, default="auto") + args.add_argument("--prompt", type=str, default="What is the meaning of life?") + args.add_argument("--time-eval", default=False, action="store_true") + args.add_argument("--skip-rounds", type=int, default=0) + parsed = args.parse_args() + # parsed.model, parsed.quantization = parsed.local_id.rsplit("-", 1) + parsed.model = "Phi-2" + parsed.quantization = "q0f16" + utils.argparse_postproc_common(parsed) + + # parsed.artifact_path = os.path.join( + # parsed.artifact_path, f"{parsed.model}-{parsed.quantization.name}" + # ) + + if parsed.primary_device == "auto": + if tvm.cuda().exist: + parsed.primary_device = "cuda" + elif tvm.metal().exist: + parsed.primary_device = "metal" + else: + raise ValueError("Cannot auto deduce device-name, please set it") + return parsed + + +if __name__ == "__main__": + args = _parse_args() + deploy_to_pipeline(args) diff --git a/wasted/disco_test/0.npy b/wasted/disco_test/0.npy new file mode 100644 index 0000000000000000000000000000000000000000..a53de3487455b8ce9ef702678c5ad7dfa228dfad GIT binary patch literal 35968 zcmbSS<$K%M(=8-z3Q3bRV7Az@En|{0Gc$L~%(P`@ZquM0lO#*FY)iK6q|6N4Zeh#J z%*_4e{TE(+)BSKCX`Z>~o^xg z;uilR`q2C#IDt%P}MlXJdQ}Q9FLAPZ};_OKZTSP_ZWT{4nUGee|W*o6Dd4`l>fGx|mx63RCf_|_VC5N6e+ z*oL~pun6JSoHLqYi}Z#-L!lktJ6vG?#jY+}z;_c?z*VS@u@-{bcZs8r#e5}=Otnqi z!7UfYVC$k4t!!W#R$n}A{=je29p&aRlMOw@eWpq})ZEuH+0xx|K^&Ux_1`gV;N0L5 z%R7U=bgX5cImgh~(wk5Tv(jd852L%pcCMk?J3^s(YS|C+^wvk>`q)PQx8M!ya8r($ zW|_nn@_J*I)u7?giW-CngzHN2 zm`VROyyF4!v7pcu8&7Mk%uVY`?rBu-&NVmXCh=Lun&JiblK>L{#G8gImOWw{Z$Q+C zFN*82zhd1?i5!tN&ag0gmr|&^DJ+H_{3vvqxzKzG&eB=4UJ7g^C%dLNmVx;x(KbH2 zZ)N;4gdw*M7&y zQ19Twu$!nNb~I;&+wd6{T|rCt9{i=HN!i`3+SEWxcLVN!8nQ-v3DziQ^qC7ycOxC~ zoKV+rZ$ssfO$De!vg=n|WtOrtl&^FZ$h3H?%g~ zO}`vS)^!M=;ws{50JBC7@z!nNz1Ves0iq$_D{@VM&?WXqH!qNlqX#R=yYSbAWVcr{jNK`Lf zt$$%QnM0<0enGgqphjfQPDB@DQg&9LD!DMy!ThvT#!39sj4hI0YUdG^Jr#muO5HdZ zN)6|T1iwU@r8{q^YpoD^Yc7sPIgPG)s3yHp;E3^jF#0+4-1Y-f;bkID-ZKucHV*0d zZ}3fBC+>*l651mCH=fR%4+OoWGu@e@oV`j5w&@&ehSi)uxx!sG%$V) z_9prfPsG8&oN}ZKfPAE1Rv=9K9fI%nZwVgKvp8Q=cI!myXHm zX|7=Ph|l?^o)0`Io%LjtVRMjf1G7vw1o>D_{q{X-k1*Jej=`b*TQ0 zr0}RQ#dt#O363qB6gnIYijl}M)2i$_mT3k#|3ql2&(vY2`o?W+22oGE7_JeL>c5wL z(M*X9jhMro#fuz+m?CQ;{l!tjtI#S`X?!IbgBQ6Qrbqe@v2i%=96`LLPcX-$TaEvL zy@|QzEWb~vpgJG$Soc^M^AF3F=-4c4c#vhB_&RH-@wCv~)+cf#)F#x$SY7+vSS>V& z|4)o*GL8R+(#!|KxDm|00)oNSdaJnAbV|E{e_tvXUs~1>*DYPGH1}5MUi8FuQ2Zin zE&CDfMiiRuOs=_3_$&}_Ic~fbT`p7;uSR~d0M)dNHg|whwfnJQy2M}=-J!DU#&3qf z;zevlc$sibmtq+yz6ajw=cy}$1d(FY5$EwD@RE4kn3*Dpv@!M!zY!V{iSV@S#}?R} zn6;l95O`#MU8)WJ3LMab+(u%rVV&hu_SAqZI7=7Hs$}VJn7sXn(B1qY4EaZx3StB4 z8`z~#13}IFGVG{1~=8+Q>KBHx?wRr)GqW)^S-oNtfzJ;zTY}nKcp;ZjAt5~`{HZN2K_p$JxBcr2WX9YpGc)gaBSMrRw_S@-kmaCbdw8K|#k9UZ304YAEWOiLP& zt+04Cu#Eo}iBpd>KFihxR|uQUdB#fC?Pwo;u5o2-C;o(~!l&xi8h;utnA(@V6o#1} zT7zs`*lmn6=q>w%b$nM-34Z{&Zeg=(Fc0_#TwLTC-vc>ZBoC~Jnh>jy7X1_M9PMv# zmQnP9oR{Ejb1ggxs~hEY)2x3BpFMB^_Q{g9LI9Ya81s_9SM+PH7WMcbY;NWVADX<7b8Y;Y!5V=r!CQoJ4$Ku8PI5_o2$@9#tu(LTL!saL9a4GeW78l$7F``3C`vOc z#7O*DC`-``-)9(FtEIV{mNb{KCg!tP%W^TYKXO@EPrN1?I9mpaO(#vu!vlPGEC~j= zRX}#JL8g`Er}nd+)9cNqSe&6(c%S+|^dNtAD`%F8l_LAiRrNV+pU}$CGPa_*h50d~ zw)A5i{4Gts>1bd!V^RFTs+o>D4uv`k*9=o4g0`AIkCXZ1m^tCK=6b=Oz6PPbXvGk3 zcpfaYHWP;N9m+7NQurj&(V%X7G-zlYs%1`&9Yh!5#X@iVlC`N&**}q7tCbnQg-;l2 zMDsb6kn$D7T}&H;HL~NQ{{-Vh+UQ+Pckhg>XR+Uqi7z#bHeL#x!1b0?>s3CP=z!i6 z7m3>~$GwC3-9C!AVt@s~dO!o=o9(|WF9l=tGkss1Ztci}{$=5~&_}oh(S%%Sm<7Ev z{;laxJTNWf8(J=KA0?EFEEfjr=%K0<6ZbVY_+dRW($)%MN8 zS_(U2-^^Nk51=*I$l^;s^A_Qn=_WnU%*U>obIkW8|Cq;=^<%8ry@je-BgOedTk^25 zEZU1O=m>*bbn=%4#nxGY?1*28!Mi-S2T9W?xX)= zX&-eNI9;*jl#$PRA8soKwP8aGJ_#RcX2X-vCj=l)5AKb2LUXJSc%`|gE=ky7*csSh zegkb4v>^w+$T&W0lJTr%V{|58y`-YPg4jSt8q3PusSO zev5A=8fv$RpA3^UUHA=X$efz>FtE+s)RbOo&x&QW!#IBrVFUX~yFWC+#75e|x5Ihj z5YV#O1sF>$%Rwd-7U?Rv--sbY(q>Nf-8yYrcwAWasn=dqdHG#!!QVr4#8G2fbBUuU%|}>2_3MuBA=JZt9zcx@cib zyV!Wcqk!Dn&oI!qH1-c#9LtQ%HB<=QH^)c13JKA9VZr(b-C>r}_jo*P7Fy~3W;gde z?)ca8v2?x>5`G5z zTgKyigx|5ZfmOKCd{Mu^J5Altm66pn^qh}qI|&8u;mC1TPN$m|>KbT|;c+U=28t=# zmzIgfqlUNG>?m(Vte1tkzKi-M(dmY+Cfu|wq|8nb9b&Gbfn0;H&77=)DG?h-$QH@-p&;nUK9V)wQzwsi_a1shI0J12v}&QzKH>RGcgrg zj$Y$+$^e=e%OyVghY0<`vqA}G9UB)4MAM0N=6FqlrI&U9z?gRtH+=(y0p+_&`6|BQ z?v6mr_X=x>71UT8^%H|kS-Ml2PT@>Vymbz-NcGaR*6=}77G0wcvwcmsOp}PpR@ht^ z>}>2U^Z`uiiFeQFk-&-i(x;ZGwX!j+8@42;ww7U zR9_6{rA9{*d$4Clok)n|(Y}$_rg(Qh1A>(WpRzacL}9R@AD==2>>zxRAt<~z&lJ`X zqYRrOyTbE~^6(Fh1uMX_sTtZukdGK=-eZJI7x{YYp70V&2O}fx8@y%EfK#G&Lzqt! z7@v`CE$-A8XtH8ChCJN>)r+D;%QqpDAESvAIy&YBS88R#V17~{6-Ds>h>lu^VuioI zv8|z-H$t2->cR9iXJ&cag1eFC&3^opl-%k;8vN7FTQ5vrpi zhBl#UrDwTQ;358+@hy9y@EOvAXiC=A9S}22J2DsfnzJ8phOHg(vo(o~$SCm*>&%=Q zy%ek&tHf>AUkARMqC$UeLAW_4BQ6UI3}YC(rjg+c(OUfE|LNSMd!ySTJm+eKkB0n2 zxlY%gGErI~SVQ=aFrXpJbK$S>q0o4hod-k(Y_7YEx(~W;LGVZi@ zkvJSl;O}W)@?*0;1;fE$@F}q(y205!JX3$#qO+VZinacOZ{S~vYbBjyBlv*KB%-7!KIjUEmzo6y#TGyq}bQ&RnOM!mff7;p)DP z@HI3E9O?0~v1F&a%yCnmf^S;4dUKNVrE@T~7;5U83BUIwrp+(8p*?`jOLqBgQ;WdK z$Zg+B>jTik?A6TlT?KY9n|!DJCNHe)tGdSAm#n6G5J%9csx>-ScNgij;j(x@+80Tt z*F%fcOOzcl-$K2(yrMdw1sw-Z)?}z(u@5yLa3AuMZJtd462&m0uX;Y@LJBf|`vm_x zbOU--!>3e0TwXi5yWpZ@C}Z-sPXEH5D4fFBoiCg@();Wa!!G6+oPiyuzVi)`0C>v@ z6Nl_^$U?RY{*Y{?TxrV&7s@=Y&Rni?Y2Io|>3&7J9L)hcbB6A!8R1>08A|?57O}lE zX1OOJJM5qE0zAX{Cv!PkOOow6z{{wnLcF({n5 zlZQUyj(K|m-O#%}it(gpmi&9PZtIBp4n~+!D(eA}g#{Eo+YAE%#SSUXkwxoY8dF5W5PTX{{2?=>>eH8uaHI~9G0u3~a!T{G)yP9hNU6$(%n}V=;s8CzO{Kk=FxD1wF+o>}>W=D!;$O{t@+^KE>g;U-_aW!Pqtn)9e5t)j zktZ$#mH1Y*SN5)Gyr!R*rdon2sFXS{9mvjOoo+7^pwf{Q9?}1;sJps5xEfL*7tr?w z{gAsrLq)fOHN*zQgeif!cpG;+NtSG+Z;R@S=e(y0J`|e;8{E6mfcq`IhWg0fRb{Db zIWRPye2v^kTBf(m2p|Jh$2N{uRCcW`RFYNL$-qPDb5tp7ys1!`L}de8OHs%SJV*4z zF#i^8AaIFXqvrfl2n}>swt=@hCBPeOy=s)_2(uL_^F4%3Xnl4R3jurBwM=`Z4kX2V1e;z zLEC@W@D;6rdvsIsSCRzI<}$#m^d=Yty=r}Gb9RZRs^_1;4CFgqk<{oDm~?Gbm>~_& z7&_ly4{@^>nQ{05wSig1=#;Iz7VrkWRK?i|*ApxaOim9JK%AgYlixJ}{Hym&~(+I0^nbCE!d& zp0a}#ZCE8gC*`;Ow%5#jA`#^mWozG1?@zD9F^IBg`>P_r7od-<4Hre{b6x#+Kyu4E z=QiLZvZbIBYnB#j6Y1*ZdhQk)@=io5DC^pSMk&8kJRWTVS((Q`S5JLK zB?QBFY7Z$4=w1@Rj9fqO_mZ}%g&CE-EdxAu(>aT)i6kqsN;?lfjnTYdm7sY02HG!q+@0)-x_bIL5-T))6a-p^jFKzRbAjX1A5b>Ap4750*gW4C`k^;~XaW=A z&fc!nXY_7zcUd(>8_x^XFlk$;E7qGn2oYY?*PogJ!(dx>5_Hj*hvs_EJFmbsphx-V zRePWmg&-+a-cg?5<_G4oqcoG4znHJU#p2q?8=pY$ayjI4JS(}{*yNU`0$5w#{<(cR!a)fFq)e&CF#A_Pc4terqUREMahwu7f=7BpOXwM#m znhEPm@_}Kh?)JgCVPzIO7j)1A(LA3D=}onSdvXN$g|Wh$l-bBsuUCE^&K1TYVJM?G znMo*~zv*l!%U8jB7@)|T=u~`3+S;NnHpp3ygdU%xd-)CZ$4ix?#4im%UGw&0>p5SP zY}=UY7>v9q`PWwstphw%j&xSytg^8-g52Z|pm!yImAu6I`>wg;(59YINpFdLV1M)+ z`9d*OJy+f?@Ip0Uk$4pTH*G zH)K!x4wBAT!OLKKN(yg&`F4Z@423_3rH4ugWYqVW^`aj;G}x3 zs;TxSxzD|WO_l4xH*9ZArb#0c>3gsS`HrkbzA}w8r>HyL{~{Izt^_zioJKET8h|5^<`jcf@LuruaN?f*p?|%vu=&!vp4pt>`3%)z z5}7HCQF8;`;A`T41C4=`U?r&aDG{qmjr^CVr&~fx8L4YFwaeKMdzJqM=nU(fd&%Bd zg5?*xTWumYY97;ndp|HUsZ^IoUKTfBS_52w=&&|y0ooq;OxE^gGFtZ`INdQCoS9lk zCNOfkn-a&ypfwa(&>|><-2*jOx|tj9&Wzq0N;zNja?2CN4gWfHnabom%cN4J8|~>7 z_XIqJ;hre46^Z9GXrI6pcrdPX-%={vuu6~@!y|!Ozyjz2;<4?9%iKfAIN&?-oUG?- zUu!x2ij)-{E%-#vqAMxuubjqp_IkZQ{c@ox%4Qq}wGIj_s*)~8@nI#Gb zZTUjqqRvWm+&Qj-x{LP-cY#?1>3v&V`LqDBu5R9%bbUjAW@=_%O|g1naetc`?Bs}Z zRduJpApV%laE%jQW;{S|!UlX9Jm1$F>O{ron~`W{FK>O{2lYep3sg1Q(n|r;ly3tZ z<0y$amyj!>rl6j-KhiAc+fsDv* ze@8~*rZj`_Oz5!8%Jd^>7S>!QD`Z^-mipUC6s`%NL?Q{q@HIA4w~PsJndl;a8oNPL z4}Sy2DO;n*0vIxpctsZ#l!EEN8P{1Ro~1@Z9-slTAk|QO z*!>8e3svw>!IpV3dpFNW=hRS~Y?7lPRghdA8B#LJSYBx3zO(D(-90hnEC;jmi~ja) zaJ`0WI=>)l+*CaaD=1ePyAj4KcN^KS)LZ={ef7X|B?Zq{9M&kn-|9YYr(*(k9qi@a zZJ$WB_Z7*!0g0yqkQ*H19;mS)tu-jp5*wXyANFz$wYPj0E#lftZ6n_*uY029{O68* zS8{QA&&X@Xqm+~MDODZqA$E_V6>xzqgP$RtQA6OBYX-j0k03445+qPVp?T?EN3BcQ zME1Zp!Bw^YFoU#a=W1{$xTByQQimO=x&>ANs~7ewnTO;_cB+%myUYpnmADq2h^I;( zN>!Q@{!aAQzyQ?5?2*l?x!En%9z**Aqh%8?ou-{v103*A7IT5Ywkpun;6C66drV!2 zx}%;hf8e#iJA9kzr|PP;uaTFb05UW$2Va=EirFr#LN>z>)d!jQyz-ckL+K2Uqu@RK z%Q=xr!q02otGjS|4TaGNaC%jFf{~7O)zwBCt+brrybI@Y$V9;J(_q#ufflx?g5%a=B8XsE^=iTp*t5;;YKgsNnag z-?9H>vXGvH~-QD5L<&@0(Syf5m zV}2+GNLpzIK|cUH)dz}#d)c;lnJSS!B6)=_m5hO3Rj z=cy9WE6!@=H^m5YW=UhHp8Faz-@6>s$lkECxPJgGlmt}sS!n_IP3C%^sT;xfkRj+y zwuG9b-Ubc_-!lv8&6z0Nkg1}6Ue2nQXRP%nF(2r2%n)Z!haGsW`WI`M(uujD>_wLN zT9Eyz#RXG53njV0SXnK=Mjmq2lP*CzQhmYc9)UJ1>jU@U`OrVi6NQffAvAycdb(&T zcmo+uRo~oO!9cInCii$v6{Z)v+ue~&1S)CXV~1q(ysf=s=m_(O>8$RFtYeQd6C_G{ zBj|FxB=3>yGG9>zs(!A_^mk-8?vsy%%*Bs}8lMBq;H9rt`cZVGL$e=>F1qCT8`B0!TDXn!LDsZnw50Mp^WRF*|OesaC zW-e#Et{=4@s||{HrjLE5$Lji7LV~ZA8RTuQHnQB146Sfqwb2<&?Kk_mm>HOJ!zvkvXd71wohx;*r}bSaBGbL=L8|xh9eldn;xRRm&H^LdZGWS!N5f zQu>3;loz5(qihdPXD!q$ zaEpJE^p^5B=~mDMQ&gY4vi!*o)s`;oFlZWB0*y(j%Zx*NtL{$J|wN+A@v!D5vbu>2VJM;I@;9Ew11=P zX=w$a@sEbs`~mC)BupE7-JPQ2IRZqkGoc7VbwjtADK*c9GQebcnu6)^lKNJS)0> zNGiLxDnFwU%|PljFkUs2+Ly7|*+M45HFJAHS0o|dUy_5iSNW6|FIO?cXaz8V`G}2B z9JQw@swg(0Pm2un3ld!`qD5RPm0GUMC0kygeZ8X?HGS6gG^4L_RboA=0?cT-YKt+xRljLb?`|7YSblsHQpg8Yp zuo##pW7Yo@YSF4{g?hawz1$Cs*;~k_n*dnS5_Y*b*T9eN+EPjbxfCs9E zD!o)Z@Faa#iM(h&+fR9~`XNQBbD#1FIA%=``Y?D!lJ6MczA6XFQ4+YgGgL;^k=J&1 z1(&#nE1OWUk|FYyzJVT2lcIT7RO+yLJGhp@CwvFQaqwJk7wjl&@@%2@ky-2?mjN44 z5!l!W-7T|e_rsskp2(JZ4`_EXg9AGGVSD5B`sj0*bmOkQl3wiXlm^uzlpYA_9zeOz6Wb+RI6lIu8G zT#iT&SOZcFFu*=)2e?3aQD4(LoSncfP^T9S^nLW@1Dm05-rD4JYtYjTq8pVZMv zD!3IKBh{6;RrA0z-m%~??>)~iCdKy7U0u0YnTX^uar8X*Jz%OiRq>1MtWh%KK}n6j zumrR^eUq;1tb%mbc7diNAHD6MMnSo`Y4IcXZ?Xf^UYX4{rT=29QTvvSO?OqvgC;|D z(OU#fU7^an|I%Nz<4T&63quZ|3scoQAM|CKGQz+L(v<$q*8w=ErhWTeOB_wHUh*wU zW%6`24!D#~asXw;RjC=sShUPr0QRTVs)%hq33=sty-Ut`Er1+If<{BVf*ByAusF|n zd&<>xvAQmk&z@%O3OP8~e#o<9^JiC%X98A>{F1>Zb;#WSbyIIf7UmBkcX;&$3)pzX zpSpj`A2d~6G2=a(>H3b2^qh7Z9S6y-?j^pk?=XmNN%2G!2aq^$u#A#-O8f0h-0%b% z&AnF+e$s-^xI*%U9B%~#j_8?n3%7C@DR;n?sUK?d( zeI%p-!0J@CR)tM<-%w1OVsXnQM^8c>^UNI95s3dX60P zZgW2Z?9vvB{_4ff_R1^d5!nwQgsqY8)y!5l1t%f()PH&F8EViOnm+Eza5H9qVP|zd z8zN=YE>eSD%*cZuDISBh+>Klt96Nmtv2V`mXiEj=NiI38d{@$=U@>?v6Q%n|%9suA zpM}4u>X~uC2k?Y+7o08aEcApsmo=nRDJcAbIKpnm3DsB%#~!*9z!d*jI0HFKHb5?8 zWmFIMF8Tuck$sW*1G>Zh!#?+X^}y`&@H%M=%0rD)rKhDzHy7&vLsxLVjxW4!)prZRM^-zOSS}m%sX`cm-DOyCeNxMVOvdss(s^7ZSGp}5; znAX@zjk+7i^l8FNCt;F& zDRZ7qB@vkd$^`y(CnEKL6W~eiD!kPPc?W`2CO2t?)x-2pNjJ$uDx> zj%G7|;BcOtJfgU4ABxQ=xl+<8boet_p;pjU#;t!>7@a7c!qFbTk(JWNyd%+Ba4j^GgQ6T{&RWNKiq`c#P1XV|B(wSQp zz_Y-0nO~toDx&yE*`)25oeGwaxpEcn+=b|0aJ$09?WOc#)mEwwZ38z@xzL}QlU#Zr z2<;TF0)V_`Srn2WpXfg1V4!AscE8JL$>cKMyd521Ga}wOn*-cr%^BBQ#X{wt!Y@E` zpxGvbx=`*})MeiSMtNJejD(QK@ENZY=)ttou0#slJK<5Pjp|j@MyiuB5t>NO0~&Zt zs$CG}UuV-x?zmlaJGO;S%{*9NN7)JW1mfk#*iX*p&@`%y)n}gYoi2A}dKEVZ637~w z1Uf;on|_GaLaw+i@O(cC&-0&A{|8P(u9b|WwO~VXFCv3)yT{Po7zyLVhv*8JQMSs? z<$%Cmb^3)@l4kC-Kq9_J>GLQw>Fg~^k31s#!>8Q~Xm#ZlZ#kWo&dF#J-MlDerC;* zvD(_UGTD2#3(5jgCH0t|XcAY)yEdOtA1zr~9u?j*4tyX_gYuLrS1LS82|xk(mD-~) zr}Xy1SjY6%FelacZ%cLpi1e8|-5XW7ko&HKB~i@2IgMFEHA4>o2bo*QcZHg%3rq#e z)cwGw9yj?&1v=uDXKZSw0#k||Qs^NgSPMOxOG|CYOifm9t?K8=f8Z1*6L^VKM=Da^ zfsgKq%v5$J_&`B8`}!ua3mtP9w|+6OlQFt3(IR=(yHsH%lU-)A47|yYpsO=M)e>-; z?KFCA%}#$eN0w`zuY~GZ6!tVO&{3p&h_Z-T%KYaZfzP*3@;9PtWuAA8ELS7D=~J*n zzD-fr*@SwoRwTcNE3ns-0B}jh1nh5jTZhKA5mXZzrf1=@q8ExcPE-Ca=1I+)#aiS_ z$zJe=A`Uzb;p(Q)QcTY7VZOOWu)~18wui`e(n!5sGtSKbU6BvSsr)10AfSbR5xm~{ zg!&8Vk+GhBCAlX1o3g8Xhm(T-0>Yo9>(4x4b~o~ak2$?%-P63byF^PTe(>+Dmp`b z(@M#sOe3g)!UuF_qMpZeXSt%Jzw?&+A#{+Q;JHB7lJp{XmrU}O*@rL)TcFh1uR;Bo zo@5{2ENYx3vl-l8(p=?c%BdRli~0c7P2I-mC|=cyIYC}{xPuUHzDcU=RsiRy)q_$Sh%3VBF zDW$x&Ldk`m?l$BX@3SrSpslJb#T{guHy5hNETap_TEKH>XS4(Mj)`EqnD%7d!p%T_ z{zgw*#Z2c6C=s0xquLh8uae2kTid_s9u4cuu8v9mB8%iLidM<>@bQwQa*g|=sG_t}6#=@q}L@DZkW zj8%@MRa7UWMXis>L|adCA^x3RNVZ2_;z>1B$W6r`AhY&ovUBoM=so(%a~Uq&oRF81 zyrIMl_oezMhsn;sKKh&FIIvYZLVmP%Rd+UUDymTqg43jXWeuP&p7s8F>^@{wNroi9 zT!mCuUO+pM{oHL;cjaH5Gl4k&P_ZkpgMNc1v#EfI%uM>=6%Z5BlW7TF^1X0HH)uldCe|ywi3ZS3$Pd2}Y)U?L zHsI#jr_=kv=IkFthYWL(umJv0EnfOmcJOZbxf zW8zNql%iJQch~_oW>W0CWoNwCbp4fWTmx)dfJIcMHs-A8K0`K8ZS*WI$X2YQPh$PN zN4H$}6oMFHaWJkEY-^^CjQ}Qjis2=yneKnQ9msez&hx@o=KaU{J?#0;3Vcu!$Z;bzpX`e6FnMsiy3T)@A#sG_0J}ATyZdXctfW+}Be&pW-PkBcdw^3j6>{opO26}_(>)hodO#TG;oC9=Y$oUeD zyS4h6$K_5#jwZLEHH;FOD^2iv*ukoMlA%tglcP$27%P*jb8{dW(!J2=zv@0;b177_ z_6n*f^P6l1G>j}%R#9ACneJ)M7I{?kf6PHudG}2EVW=azKka={1!r!?-*h#*N?8ad zk$v4mOZq4(0M)56o+r*9KquEnfJo_(TOc{3SONU2#J%k$64&IsE((+MK5UU!cNenT zrN+uFG9IZHI&$G|zHam!;IwOIfggE?oXkJuO7QGp&KCABXJDs*^L8Cm6RzPYvhCdB z1Xg8yRrYYBz%b?ZlG=0?^-h^B{l0DP=JApmt~mQC#}nXQ&8{0h0xQ)=J@457*&BKY z)-N&Bw!#n83^@eXB)hpAF$c*#@KTh?E2NH-7d$4YOtsg2K@}*tA06f!ZtqoRjbK{Lws#XmOMohPZj@C}#JO;HKp4)-Ns zDsTqwqa5UxuXN2MBs$~}zQbO&;fv?6d(-qCeJ_MK^o_JCWt7_uqT z%V|kF0!~3iDB1Ul>f?EX(J%qjHso|RF8k)B_T>~sq zH1)d5tJpn(&%jo4fp=`JVt@y-3QmFBrS%l;k*3ZV+1-Kr=gVFdUnIL^c<3q_4{1OJ zu$$eB)SvU+K#6KOg3%Gs2Gn|NQ1{eJy&m8}Zbz>C+>^UJ=jcJof|UEt_UdHM zBjrV){kjM=h@7FwQ+!f)bpz7M=pU9UyrDQ=T||>hFKavadj_MpF5>Yn{TU6H~Kl z=}+rbfhaS;@|SUN&PyF{u$XL$2iimY8SN+YKUp4P0eUEBt-v)KOX6!Ukuv$D%oS)%Zw84L3Y0~U+GlgBX(62Z&q-O1>wKZjkGa!lJ+T+BW~AC zFH1MC%WfY!r2nSX3zsb^v0b`FvCq~v&R=|-B~I}OLU6=*iu-1fMv}24OR{AV`pWpt z>@97^EhC;#Tm4hfdG^gJ(cIbk$eigIS4zj;i(J+`>*46ez)o?MDIVV-p76)vzXheS zG=k|4Mry?3(R(~$+Mn%+RK|rDg2MEIOpPxR6`Of{iFqT2EqZ)NCXA;e@u>)P!C zv-KS{!->^=T|@iqIou**JK-};5JI948zP<*BSKI=!B8)AV>T<~SOoKf;2HCmz^>p? zOKSN9>k(qLwx;@m_#0S*9Sv#fDD~GdxNLyf-C(i0%Vw13<0F07BlW0D`d2!R{}~!+ zN{ObJv+EuPc7vBAE28JKdvPnn{|T^o9iKqd<{n_zOxvy7m{a0(VXLJsaVOZ?Ff&>; zt7&8^H{a;iL^#OcwGNI=ur|}$Vn5Ah>=V&E_K(<3j+EEo%m24aWUBFSpsC?7z9iJ! zvYkJwJA(z8uewh99L=H#%M8=1#reAK!j13(Ly6FZmI$xSXsjbv)7v=vG?8ciYWgos z<8#aBFqrU~VvGA<{)3+};0s^M2<8geOm)-1qOg?uPh?)#XCXa8W!P{3tB;?)5E{KU=rS zwcIuF{E}(%!ax#wSL%$mBsZ1(U@GykT&4*X{-jPQw^|xq_nP^1YDk(ZBQT6ljb;!l z@!H~-=q==zxkGA>Ethx@Nw>vNHsPpTXun9TfP0Al;D_WXa<7barS5Qiu(!p5LBVvvNzP!V>{w7Y*1Tpz8`xppg;%mSCpuVG zY8T?$gBwa5mYZ_p(68thk??`sR4R;Tilf-aRHF#G-4*h}w%ufNs-f+Qqm<|siLw*eB)KXnl&leUr1pe5kz>GrDJf#IY49AW z1{NbXBpTSpK>tLZMivQ|;5ybBWFLQoT3GU%*#~vS|7ZPzH4LcIDocGx50qim;y3WK z)O2gHXLNLG!NIVP--bTN_XIwt$rdWIJ~Cb?!*s{D!`&^Ff-T^TNSI3VD;XnvRN7DR z6E!kzvrsEqjNLZlk+y~-k>`RvVz#O%25)V1lgVs0;fe-`|D2e-K;DWs3pHn~k|mN7 zIxjXX>Prn4QcK?1+L-3Cqr^#a1>#RwOFkmI1aE|J;zh|2;lI)aCD*IZq`Ko8esZ`5 zw#CX4lh~7yI%K}}K5Rg{OXc~2*km?4QZe)oE8FHTgek0@fwEGMK|H zm;5FDu>(X<6E*IP*)KY&P-I(OM)_1r_66k)^}%2~Bjz}L)YdXQm+2j8M-Ir`hnQJP z9HkN2()8!>3S_(=u&#=h;bD12dV|#c(oM=9?Gc!-`W9(#txfLc9KkN!D&fZV`9!8| zUfNIa8mgmy!eh|0a8Qvz%Svy=#o=GK=U#PHrGBiE?EZ049ROn63w6B7&9Sx#p zW`nI)q%4vqE~jEqQQ9ipXFHic2&37-MePY=TEj?Zv9D|~UrVnnb&(q>RI}T6m-28H zVhCQ6=CO_py(odjih^KGuuYA0#wH033pjX2q~iKJ;qjzw|3~~8Y0fRddSWK)aG(a> zP5;&%5lCAiK2!YbS&3^YgE<5KE6$X=Q**?^NdI~2{%Jq@%Mcle%@{!ol^OZDNlc;oW6A@JW9M5eFI4_n=Y~Z$F z#uC5=nZ4{kk#dphB15&|*V?AC?c@afq2(uAMK%ztAc&svdE zi6KiL+m=w3nk+oIWT6=A9&N774YDNJM@ma61}i}KM~8~%!`sjtGSAk@rn4^+*D^aT z=dEvXH{8MaANWyzBy6`VOaCg()zzT(J1^4#s#tz3w-VI!KVpCQ0{o0L5fj)n>ZtTw zo+nqdwwHf!S(bF1Ay&iINE^NXO{%7W`L{{)?KrNC&h5pzQvNY+dK8a;${CXYuS@LLfFc{Y5f1W85N3Q}C! z3QIScz^YijTIxkl!i|Zyp<&eHaHh=}K4U16o)uSMKLs3siT79&2v<$Ztv0i18(BEt8NRr4i|S>R;7Vs2#PL>TI1A?j-iHSF$C^ zpNMPL8bG6ndS>}{1ipD}+xAS-vu2D#_8iYV@qohSMtpoi2v@FU(xulV3uttrD z%Ge%eQ`7AU)=kkhL{DUX=%Bn_uH=+;2JM$h z)V2deQ#KS$B7E{9!pv_F8sRe(Hkgts%2w)l^dVI{SN= z5!>2cf!Dcx)LZ*sqB5EsXyA`X2b6Vm8lOuyMc(o8f!y#isw?TXb+l(VTG$}Dw(F^! zkMCmL@;>WL%WvdgSM|uXz;nDld#7}v@B`V+4igvn<~!zF)(ZjSi^#!9rs9BU9;ij8 zL$rNFcqBFkZbH1}-kST`nuWR%X{pyE-^iBI32xQ)-~K_0Tj;C#fBcT{`jj(~S%E11 zg{X%>Y3=X?`!=zy&>^rim>^HF{HGr*E*09^cJiOo#zvP$EKGUpYJD5x0C&mVPpm?| z5>ACXvCpX2czG91ru$0LzF1y}54W#`aQh0o+4Kkvd3J^|I0;wb<7I^jEA^%{vZBhf z_zD)VZ;C$?uE{;&)};=!K9(k1%u;qZB$kotAte%FzZDTmPT2ycD)?pj1Nn*UOLXB2 zb|hPOk%{4D_)PwHw1L#4v^ai>J}@dtw_v zzA=3w8ak)2EuwkyIEo>Lqi5x9`q#lh!OVz_I8Hnxt^u!X?j7|?o?`z))!0~TY~V3s z4eRJ|UYY~Yd@Ci*khnmskeD~#-JoO*TSiz#3=OuD7N&eA z7ZWkuXJdD3ES79*Me3r7#1-~w$s_(V_{{f$U4vE+m(*;B`H%|EYT>shMhd}oeP30b`v-ne-2cw~h zco2T-iMLKgE6SY%h1| zkHgkqXoOCU9xCYLrP(XD7EiOVw=GjB0+v?PPChX-%vKs6&x2SWbR?I8 zH0LtIxBcd*+LIn_tgy8eJz*+|n}$vd-w{1r+2~iHM;NDa*%Pcrcqz>wl9-l}>h|kU zytIA$2;vr)XV%!_rFT$)6c^4Uwo_Zguf%Cf8}TnaMq-7%WaW0z%#YXtRMD* zdif~(kGrK@#5t%xfp0(;{x@(I&6D4gkC10pXJn{vid+Ll$nKi`me0Bb3u?L5u8fk=uwkapiaMfrh^L}Wz z^a|MIR}G~E%G~jA@YJ~Vw?~0)Gz5F-oidfIAlL*DTm$WC88&` z6FCGIp?{+N&5Zvcs#qepz05B-7}^c*gU*uu!+o)J(t55e-Ot=tal8q_DRzIDAW>mE zZq$9J8e%re9oZzVG7sj)Xxdpq$xDR#rDozTb(H8s^hNT78-kNrf~!-epeK>6$iJaJ zMAToHcoLlJTSl*N)uVfw{%3duboW8fbQR@m;mh!c6D{;<$1v01oH7N+4Ywf!I}<#s zJqq;$J8o`Emo1!ST)Y0Ol3pwE4Wv`i4SDqpyL~~^0Hf8>Sre~q3zj&4r~JqtXfnif za5mEah2Lnr^eX_+&!e}o*$Uq?)m`3j*?TUza_p*tGy1B|V&EO^*XcY4&lug%cu{M1 zmg=?x&%9ODd%$Uc%WyzhJ+HH*P|s;D=_v`V}QA2ZY`cm~!dLz?}_ z#n6Yu6c=lJYs}OB)xI(9s=C!e*2C4dxCI*Znp);c-jS|6 zGpH)G94qSSTE%?O>u6QZ4SF_yGvOxvx}d8Dajw(!^iWsGsYX zYtj?X`jfLaGQ~^=IKcGB-=3*$%+!~9f4HhUB;#lIUB+bC0#cR(%)OLy{z~38s*YXH`-yE%L!}qvg6!4-85JKG!g z>XTzSB|o4a0n74Dt~Ks#I?n&p&-gy(mu!sDRdf#A(l}`qa6jp|>qY)wb+SkB9aBwU z2)dnrB%K9(D$WQ@bfvpzZS<#v>G9g7{@nPpn%+ehgg(r2rj5s;T?YJh>Agw7_hizT zt@~aiS&E^bxdR=eTb2EOSV?ibBW_|ISD-bFV6 zDrrBec1OgMuStK@ zYfs*s=mstra*B!KU%+MOvBGDSf8uZ;LFi8jGm_PBGy|l4z z;U8cOUCuazv1?*zILia92R>@vI7cQz8|o_c_&+&s#fF9x&g!P6hI876ZgYiU+^0|n z(^~I7Q&Zqm{2uU$LJuXS94y#iTv(v>bk}ruIDHl1arwz~-`Y=nVUv#TEdF&KOUd;` zYc}?et^Bo0vG1>Msb{!u0k9lQbFDJ!5_@W{`{(MT-bUU($>UUl!wOE--6-zM)K2ul z0nL1Ip=ms5DeRNI!cElVU1u~81ABm$kkP{$`U0Y2xGbaYXF4Vut9st6va}bCGl2od#>Hoxk98Z^EMo#QG%4h5 zpi5K_^w&(RPoIRE+LjPkQ(8elQ9rPji}j3GTnkF?)!fCiK*=Z1%!(#wK|{@p;yA;9 z%2R6=v&nhdV={S*^`_OHM$mJ$$c&}OIFhvuXf@adY?qSkf6b_%7l0ZXTWByQnwu15 zxD4vcIX=@N?F7eMBL~>Qe{xs)(zNB0QdJu~1qFZbg+(6{^u9@OV_jYCzR*R-5Eo%^ z<()T4waObSB|g*~Qciw1*bLnq(>hsgSa0^`PIP!x(>#04bo^aPgx4Ba~51ki)-qMK9Dl7WDMeq&}m{X7>(SAkNse_T34myJi!)22p@v(`da z7426~R?$eM52Y>KXn9N~nhW95cb67Wm>{?#*}Z1GCvSCbhSBOvqKe z@TX^a;t6jtFfR6AQxxjpxuGW%(!aVxLHU8D@o!CJ0MfJWFGZs?Z5_L`qv`J2xm!-C z?>nEl0e2PpDzgjx1=KDc9rL8Po8ywZua5#Jx(ag&igtkulP9Yif@Wrt7fKo8>18~U z+tK*V&{lO#+s<pR zT{GeF+v<4Mx$Pc5v(H!P_Uj_fDj>zokMHKIWh6aqpo!XPHM_fJCgX;9{%wuDTAON7!>x_Py;>dVyF7KXB@AG`cu!3>k6ys)1b8s)q~4WeK#E%yzXvb%5;2DtHH5}7EhOg7=0g=I-oHS!A#GO8tI!S zCsbjQ^R7GggN!=^y6lNjjW8}${nq}{4)M)b^Vw5?w@`Q8Bf6y~KQPL4$Po0V(P4)P zJPRfpcj^v;c)q}la9Fih4O5&S!44)*b|v**%?Davkj{ZU$0EZ_V zt7s}_dT%(P%1b#-cl6Y#eR0cLUq3^Y@J;53Z%lHIaj6eCj7+J{-d|sqJ`i~Aahm=s zCh3um^S-Qt98bq0yH^9-pow$}V*(eZAeJSx$2&v65xA#NL7eB?woSgQBAi~MZ2%(1 z=J0#>f^Cn~2lQI&aevF~pGD{6hj|)0DtWTEUH6RB56p$sHInNY%R;Xm1A&decpsv2 zdS}qLO!MQKX>+_u9?5A?T}#p>Y zwNBco`|RxLs%Cm`(E5%tyPan@PpEN(8RA?eURT(}&pJzCB{0dWW7avIuUp~Suc@O; zaCW1!HF2I@^ip$cu(NH7+h064+3shOd3{UGOYe5L0g&m68fqDifUNl)Ep%I4?P#~YonjRD2sROyr`*xD(%b|u=(jW3ac}9Wx*Ewk zrdEC%@3QQDMK*Avyxy?^s#EKtf1vm{va%+{`4 zO?QlGBv925^^{{4m?L!{^E!v@T4`wLJImylY8r2Ak~A9(gG1FpT-C(eS*I>=LalW* zJm>k18wcf$(b#nl)SoO}pgEvd$%Ievwqownw831o*EH1Vb^kTa3=Gpp4Q-q?+-)>N zox6%M6Sspsb=T2&04_560N{1{`m;%WG-jqzl@;z-C~a@u{nwb{=!o z*TcI#ISuQx<*4UTKADiE*=o1~&2;@u`Jm4OPP-<#H)VlJ3=cY$#RckV4$N3fr7sd>$sIl80jT=0u;jQftOqSNO{Qr;>Y>WEIH+vL{*M<>5d z*3$=)GE6bdq)Qn{G7SItCM$>tDz^s zTuIlV@T22uN;LprS_6w2K{yxt(BMxtc$ev3CQo2y&@^zyxYPN+q~pHY>uc+kQ@_Ju zjOu4QKA6JbJ^GkH8TJ5W48Pq$&EA;buDRYHDU}QF>K3zcp3j=83gfyWpoQNQuXP>r zQmTPPx%qW{`z?dFolvq1h|yY91w5~7Nb3w*?`qGCqTGZzSUu(rSk*a2>vUzf_G;U? zj~f!8J_Tot)4=5$Z|UopdU=}rI(Z9|>v$$_>+G2B+il+F2GzO6EeruV2JD)D*1KMN z)VB;!Rr7aFopJabI%^D8ndyvMXVQd;UPx+3>7 zb&_K}IMV%1b86e5f*e(Ss*JZhUD<)zE6t>TY2?btlo z_0m+uJ;Ifj)xo<~ThsWR_Br!BNAhN=_qcLttK(;Z!TGQIrDw5g7(JTKPkw2b14#$CAM%cfHOh1f~p*P`<7*_vX<3+-Zs4PD@9LA!jn=_IUFzZ6CC%Q$?p346c+}zP*Vjd`gVqE z4y%8bQw!b!?7D^dW1RIpukyASyoC>o+AzI6sfJ8&nCe2gPpY8fqzP7CQAPCyP8YDS za9Z9i&(+Fbk~`@-(F(O&m}HpbALqQJ0SZes(_KY13+ct080LZ3ZG=>W;yem_)rOMd z3(78gps=g|yrHjQqTz>A<9(s+?tM~NoX`kttoxJDa7ACGQi2q1a^uA-HH}=Obi=lh z-gcUMnv=2pjgrUYa(O|wU??trSk%xJ2e|Yr6LYpnU?XQIe?P-%#a-Sy5B4AhLgKFY zu_?yfiFp;k``+J$c%m??W%Pt3iIj#i{pzHc^moiU2hyq>FtJUz!hMDW~}!f zc)QSC^xX9!=40UkHEuF9Z@pjj%T0fCKNUq?i*hHXOad^&EM-Tgb!C&|UCkX^J-w=A zrZg?cs0DY2p`dvy3(*Y%{UteDo zqTvcmYs4g7!6%66p{I0`=e=nv)>YhPUN6Q0^;taJBrq7Q6smw!5f`Czq#?*fQ!9KG z(G9&Kq$B-36~J>$l7w(&r9bpq(BtE9FmbNyrnREdDc!Bf)Q73{Arn!DI7iRQ{$sBV zU-cEkvssgSh1rTe@RdovhUMgMVI|>zYo%8W_&{M^9cn;ez5yjw!dP~L>!fb8?Ibm`Hbd*-l+$*q1VDuB7saHjtA>O z6}>He35sLov)G=lrr;4J~c)F?bBK75mpR zD3C|C7S@};3Nsz|@U3E9qO!V&|5@NcFay521;yWn*0K{F7r0?Wg0Mi}Rs4WN;P+TD zya!(?s4Q@r;_MY4>APEA;{rSoyBACYXN6?m890udbUus^k&W@^!We63c{V)CUqdWr z&Ps$d6-M}pu(Hz7ON`pQ4-W`!G%N5%QWQ(-GbH?8GN?>n;|*)8hi`ca1NSMgamdg zU8aA{)72)s-tw1O$+r;?s{i4RTK1Ax*z2A|c!SD_#?ZgTd7fE}1=))};LFfv%lVok zalzYADOMjYoNc~_H#7XS+zxqoD|#785IzgTL)}m(+%c2{^!DFiE{iT*b|BOISAfj> z(W-D3-jy31qQYmux9~$`vGIp+8`u>79=Hc@<-W0M*n%%KuJo9VS)x@u$~6Fo!%a%9 zcrJSztzS4*^QWL+c(L#w)6jnrAHpzh(`RZNJp$hE-(Vt)ojmWL(;_W()8zou90U-m59+BcCeWl_8(QN=Z! z?t)iDA3&AKOUe>bp)Er>cy{0+x81-Fr!i64bip7n;U{A&Lu{+866eXC?z*TeBr6=-Er3nDJ`L7|weY(v|9!Bi)a z7=mR&>+zb@ddpM1Z>XOohsbg@AYXWYA#bH~;Ch_$8=_OR!-JcUuA%>!x6D=L5Yk#& zWIm1*;u++2yj-9%_r@>+-z<)R?^>RfeA%8G7%FY$%WxmePsE=7e4&-)GaM5rXS!kS zNSp$5{1>-=Fn0*;6J`ZkaPz=sPz-C&vr%`HcLDzEl|!Vbq|t~T()b%$8s|H3zf8-TU2CxV9CiZ;i(2`5zL zx#OmmN;+yJnF{R{rYjy*+0Q53hcW`Ok)H4nv$J9yt}8CXMK%7gowC>Qdzsmdy=7>2Fu*+6hZsFvcJ&4$}^GYA9qDA>(%CulU$d@r~uzuvqg zVpKQ7_F6RJIjkAmky;|zAl%xExWUd)(o0V9jZwDULi#{y!cTT3yi$*#lN_@E2s&Y2 zRyx+n6=BA?$Uwy)Tn1fBj|qG<7r?vpb|{0|Wu6>rCTdy5z=sxDvV~bt4c$t-Q?05y z)|&@l4J_^O6-pL%h)|uJ;8=v74PIm`GK=s5b+`Jza{~?8%CBmPo<$Z1E5Wm&4s@QI z^ayx!y0E0Fev!Sf^#?unBnam9?tfH>!3>@0vj%V zSNu$AK3ya$BSHKhhOl)4tneVd8aI%Wq_5^{j@s-R1d@;GrU1WCZQ=ObW?Xw4#%2cE z;%njEVgWnYTaRjvd;HGeQXvsj3#V-rO!afOTgJ=7JU{&f@ExY6FV(U+h=H@+C%_`) zJJ?iZFb{%i`eh*n_|5F+KOuGb@`2d2{Xx+%iS5Eh`SQ4vI49nR%lkVD2LceYRhi{b zpFJnAfq85$Byz|2Gbxv(U2uZXn`~E59Uo+AEl)%5!F$YU*z1CLW_5TQdkvZ(Jap9c z-;Ou4nd~=wMs<1l5YH{-s=9OVHP^)wB|CxxrRVr6CGt9%JAv;K+gV?Q>-+l=I{$Zm zfo>URvfh`z36H@TygPFz^emWSvazJzDiktGKM<}Z)?7p{KdWaCPP=*f%)JmFD>u?82L?Xs8y^mex3K z7l+8aMN8C88;^T%+)+k|I!;2f$bXN?UO+74I4CjKI1W^27$%}qRk0Q0js%R#h%XZ{@gR25Y zU_82!nTTf~>qHxJ7CEUpWtjj6#D!c{Ou{ju2+0&~GVSq+%AAZ3rizYckYI5bn*tNr z;d~-etfc!Y3N4xMfqcJCy)bPJXs6BuXNS^}`)KO+58^WB2=^D!z#edv_k=gja5Li~ ze9hSh{NP+qWivOkHwkxbdT5t45>F1Fv_8ViP+D%5eh}Co#l+qVb;oYNtfifIFnI^- zYu4u_Mt`bb;zlAs%u9Q!xsSKLwG0`8DV^|saf!{;U(P0Hy^<) zk8%qTi0=fCrWyPfk?8-46np=}wERnPVhPJ{B^H6ZSB#SG@?V5J^bA)O{GR;SP+-c% zooIoeLL6Zgdq-Q3+l@~}mmxd6-cWz$3=x&)6MGpKa*EBwR&#xWmvN+sh6()>!Yp?K zItr(`y{s93z)Xcd2R+DY0<)OB+XNK-jb9MPf;aJZp$+(b=|AZaR^NY^{tYMy`mpH(%$Tle~FF|BaIhQF(>wjy8%C7|3f#!AoxMpXCJM3rCOpP$#qmF)!BRno0r5U9U-G( zkRR%=R(y==ge9?VakO&ZXNa5p+tAw50c{3*J3B);`}QXKu+wz;Ol#|N|1%#XG%M+i z=Q7>7BWq{iXmI3AOH)J6YCeHG>`H>_erB+OliZA8BRCp=bI7TATW zBt6>10Kp_uUwaIR~@FirM-C(~P|xse|SO+Q_32LHW4LdIfl$HH#}KcSP}3QxIbVIsH!j~DI-_YhjF8GwjAwMCH6pC1{I4r6DP{A;_$z-AG7 zfvz?m^k0K>;YRq0!p`nFc&xmJI4U+_D>Dg((@0aSTsTfy1=~zr#R7Cj`JdERVLxd? zcY8;0bJ4oOWqe5JUumXzz#<4M{2$r+wepGDc&G4gvK_RRoMx#SY7Th>8^;PSMLW6( znUoXcDw+lYFG58+!7?)Ux>PMA2<=0Lq4nS!{-)v;Vg?@2UDe-!G3iwT=_vvBc4rS^~l{%G&5VBBaB<3oT)k2!Fn@Tg%ZtUkzPzULq}eOIH(Hq+?aO8Jq%v_IusCXRCAHHMj9QXaneyx8#li9DCBs;IEmvnpW4RZ8x@Hdd zqUg1|62arggYA*JcsqVAKa{9$vWCtuqV$0zgaN`pxoU7sahxL>$|o8j@62NZHUxhWHx(n@sJCtc_^NDM8XstRSA2L`Pl9&7$&RDoM%BLv3Lq@_jB6{Qm zdPMOl1N>8CPhqn3&^682J3L#eOjP1t8#-FDka65Wpev9by2SjXx@vX78~i3>s89`^ z9$bzd6{Y}3;c`q8J^{voBm8|h4q5MQB!ET}8xm&GO_WMad)HpBY3MWZ)j5Kz?YRot z{I@ZI{%Y)KHlcRl8WU?dp41=u0ijH$848Xmu%I63fZGnA;qL`l^%<@rJT$aMEMgf` z`Ru=ZKO_cnArgNAVna^wuz^O>VZphLz38eH_`}cS`@xfiBXAE(u5KWn9NNjnTZ)BY z#4av{Z>Lq^)A7|co3N?O0j8dLLdsI)GS}7cP$&d4^nJNKa4&ue|HjnU+|$yX=&vCF zD;o5gt!_tW|3Kvoa5$7|S%ruFO?7$D82Ao)iLa%Y`b&LdfT#Q&AS1v4Wh`6yem*__ zg&!rjksHP`=3AataDx6OQaACiPsPtBbV?EXN6`@>q5Po87<7;7U#+1l_+HZA;`T?Tdm@AxzMYPuPCT`tR07hUVE z?E8=7`FX`_a59)^><&NhyyX~B5_d^-;0p2h`;$iE=B3|`s4lu5uYqIRj> z>LW}gq(jO~=@_pOt_c4jv*FKtQfQ@E&IGu+1})rg{Yu068UM$QIk$m8pVPjvMm@W48_7`+Z0*@uT^sRb&!pG96@5cy^RA)V9=4F z`{r559Rs(RTf&=z<@Gz59ziI$&@=`6Ybg=?DptzInq`z%m}yo1LvkKl=-(F+q4ECF z*eb%x?P9(}pW0T3)42*@jxEDEM;mLihielKtP#7*R*xdd5O;yV$xfwnw#QRLL6vUm{CAuf$NG1HKV%N3DrC5vHVq-HTO5 z62zXI0EhTF>-Dm0hfgL#(IsvG)4@S$-$HAJa`+lB!M+$T8$1F- zcoEzRZX}#E>uq<~ufpy0n&B_D85o3(6GzJng`Pe&lZQ8x|43WHC#Bo+JJA|xfelZ+ z3iEJR*w3BT*CV3dO86%3rF4?KL`i91;7BQio)0(Us8lnbr{qO1fNRku_*M3pP@7s3 zu!SE*SB1xWt5{c;;+~r1I-Y}!%q4;{udvjx4yWcJ`f!6?r(8JRrsT4{v7vEk1?i@U zVEgzK>8bFAN|2`!i@9EbYL?IPcl06hh?*9CV6F)rjSR;pqdSSrJsD}Oy- z0J};YTe8OAo!nW{HnKUC!?$N{rN(9$h+Fs`bCBFAPYwni^%jSaUbX%rs8pyo%mc=#jL%W{ph z;(gRJ8g7?uE9l3tLtz4IMPJKJ zp{MM3{EAH4%B2+xPl9toFX0OCQO9J7$7d9mTX%jx?D6B{IUG&~b8= zd@edwb_$i#0A=3l?9#o$*6;>$4Dn7bFAfr$+q~8osXcpyJnuUw4G2|B+6NCL8iZ|W zcj088+WMYqWa}0!ByL2#lF!y(nIO25e~lX>_3ZC0!?<@LSII-EW$MMW)>H&tEdi;A z?D4`wB7J+Z&^)}TOI3jVsr1Y9(4Sp@BrY@zg$Vzxr`mNnHr z0zHeb2%f+vrftTqr#+&65*hHI@Da48Y|y;I6Yy`ub*KZ=Qtl>|;GOYXVUS&oCP~fx zGd*G|Moz@@@y*JN!u*swJSe{i$H0EX16~$;OW(vp0?TJ0H}N5=M(AZKo4>`BB@S8U zFl*4p(Uq=~!b_nPoM6kgb%FXY9k^EEQt7gM6sSW?@inwAk(2l?#Atp2^ja*Juv5Ap z{Ffk!2?}HN#rrqp6z9PjVv}Yl+?Cu!)Q9sd6qF&>=gT8aOI~^3M^1Y)m(Q}8@VW3g zX@KsQuvJ$A}S9Yx${+O&eu9qU6rGz)<`xHUqDMGogkB{n2DEd z!rIUS;uPUFW}(?qWudC@GA$1|Ph{e8I0ApQmJ;tgL5(+Z32vP_*L;Jjga1m6VP;@1 zVlm_tPTDdfsdCx0+wyTU&M?v+a|+S{A8VaV>S zT-U!QHOs#+(A8Qm)XUZi?}c9%uUVG|>nhdAY>PED#oZ$`7@5t!5qjG8P~WADNKZi} zWm^Ui>xhZbdi-AO8P_+eLzW`9_*6|B^0z4u9>+k|QpO}#GEWOzs3W$5tsCv%qiS+# zC?D^iTu&TfN#o9Lokd({Yj82bTJvSxFFvUERceEe!Vf3zg&GJ`#6|eoz;Rm_yk5px z;-;xf#uPjir^zA-k}ptItu0&|(-u)Uu@!$O4zb6}J`}TkR=mVD6gT4pv5o(t{URt+ zqZQI*Jda9~ZRI`pr6t%K@{zZ-P#vuf&&G$?)l6BYIXKVOxTIa$BmWO`6|$c^P^iUD zrz+#EY>k3D6h^5?-l-b^)g=m;r4^=(w8gf*by?IRHJ-r~^j%Dl>*$Va5oG$rS z%9I<4&7uz-^9$0^Bs#UEo7kpgaAY>4QchHj_$t69vX}hIn2KdvQbCnMjSUxb;IAb= zqUG)D@jo(z%_X-HS)@*RWZ$!GwhgI=%m0!+xnnPNTeNng7Rh^>JJ)FH{k#6_LeEa%bi)=iFr` z(Ml>&JcuGlDQ#3O)hK-^A(EC_LxoUMWDzryS?+S~z30vh2`Xx7HI_CJN}rZ$Yp>*q zrKQn4p8F4+pU!!m@9X>he%|js@UB#?tf%||kuOacWjYZf_`2@FdC3=?_5A;|pVWKW zAv*;-kufRb@lw-_o8|t1l0&;p`(t$I1$L4r!x^Q?>S#5a_Xif3y{S=THe9XFFt&pI z;yUqNCBNUs7Dg6L=R_d z^3RN6SN(gP)qFF|H9j&jw7GJQK8&n_(8x@R)0V11n1X&r9_;~~j9Y~gxe0EvWz!d} z8+6lBNk5U(sp;4a4!i28vc0lB8V{1JfLABs3&K-40hMU+Xea*FIZzu7@5`y`Ibkbb zU}(p*zjITWK!))3Oe7L5@I(0-wysYLs4!V{nm{C|7`AsyZ_(Xhs?*l4z4N z*0(bBvbdE0hV%%yIxTjY8UwGW=TV*TBO6ieopx)B>e3uT$gg2<@vCsTS{OfYz+1il zQ19V1s?hO(ZbM|vJZ>J|gC|>uqL-9yGAV$4EOjcpQJRd9^Lbx9o36RTWz|tBON<&~ zduU%(zAataPZq0XX+2Pra6|oC8>23B&Skg5Cvijc=QKUmofO3nGXA34aFlosw1;Mi zqphtx=fk)~TBD40Zov_%i$xQbb zn0L09Wg&D4(}J7rnd)^;GxMre{1^CK&P0E6Uy20WLN~!d^0j@Xw)yHMa6h~`Y@r+Y zCXl6VWpZ(<{gZ>+O%_3`-tIXXkjZn#d(Le13c1b|<8!DJ;S~ydR&$aMn|552Cdlce zz?H!-$Vf0tdCK-hrv}@Fa&ZJ`(huS#dMrOu|3jJWISm~QTPk@^DB~hYjMe0#^mc?y zR><=yH^6>1zor)w=q}u!*z}hReRvs+Px0a$Jk(l_1$H9!F&e6`$AY86Zt)!TTS2kd zPcr`k!Wmd;*?|9J{7yQxSzJ5+l-n#_0jE+b?N+5*)vIi-7|lOqyE?zoEA(6DNpai? z@D93eyeKsr>B+@`vEC@h3FlQ@5Jr1{!37EdTd@Gk=@){vS{Ld^Qg@uO{{bB2a{ecq zUth$>$UNQGSjhcGUly~CZz4Zvx#~HUCRO!$cV0>&a~bx)AG#(I5wIAY5F$pj90dz) z&#FCa)6K5mlVG=q9DJ2tt7pS9RPWdn?nBn&i01VF3q=_TCL_R+n#t`zE7|k>Z~Azl z$+^T;*SEoQHnGsCPifbBhT_?nI8vSt68TqfU5QFaP6RPIUn-<^!a*mRVMS)WF@8Z8LQ{G-q~5=+66nz zH@(oz+aIuvjQ$ok+7rV~Ol9q7LY&D9$#rKN8_ddlH9D*`!f(|N@pa`z<96r~zux$q zn&7%?__>?1kNzj3@NL9i>RGhF(jYBRx<*)TqEPf%eB=|=izTX79wjUAHL@>0sI|lm z7som)%w3gZ#znl4iQy*-|N$$aH60nYV5G-~r>`%8|%T)h7JcXo;_=8K#6i zLh_?Q?JrFuzcVv}$8iZ~bG{$y@6bVx;EBcB1Z7JH39Bu=(zDhg<4^_EpZgr8r0X-n@)9=&6gI@(}| zjQ0MjZPFXf8CR-fh4#LCw*7*VMm?Kwir+)tR(IpDrRZ>7no< zTp7p*JLqRbFZkH8RQ{W{6U}Nb;dFTwHGm3B=Rp;`$@P-Y zfbZ2c`W3u|agcWsjyqUsoo`UT72s1(H;AbflwV1;40=5SRrC?ZUgU4uJ_P@)&Aw^lYyN}81&~c(v*vZaRaR+yfZ*?X#>mmRg|6}H81P3*rp9@p8k^Z#ocvb*$0Qk z{q6T(c=KD%b)Do~NzQ$LEL*$u>fEhgxwu(z^Wl-Baz{*u8yMh5qwO$ZfJaZ6K4bc@ z$wQ`0A35s(#@h^=m^-R$Ja_D{siVry5gTEsM;Po`*%V3`78?FPj;dLf5qu>zo*0U7 z;Hh9t*cE9RY-pJQu8xpIF={iPrgn%`ofW9-;bg}RITeml$^3pL+15tILt1-%f0o+B zc9(jiexnwl(<2ij=GZst5z&N*CsNclwkKMNVE^b)<+x1qFHl=RzW|rR=s2~)K_h9sY_Bl(^M?Zk{3D@ zT1TA^w#%Y{JF&aSoag~$nZ-f@nL~qZ>g8C6qtlT2GFG1p?sv9`LEGJF&)0BP)zlu-WaU#Hip5#Hh}Q{w1v1 zeZk(*?g>wHR3qkxvZyCEx9u$+QepC{ zqPsKStM`RCsu!lW{+?wF?GF0+7Q_i#v8b_!{wCPL{&Dvs6N&=5)SFCThWV3 zL%5xs;~pM)Kn);r3^}&S;sY*$N|V}WE~lEAQqZv_J+l6@53o^6y7~%0%@ZIcRwMQ~ zvmPl+1F-#QB;>WujJAdJkyqg(6hy^^>|jfDf!(dfBSd5$beoukR$(tfD>7#>4*s{; zKYU4S6kmhtPK;;fQomWVZZokmX*2*>sH4BKc8qq_8s43QXvZMWM$hhn6u8*J=Miqt>?1A)=NQwjP$%wuoR>Z?ttMG%c9BW5Sbu0iEhMQ8U;wdto zj>AN;6|f~XnP_jj8?LbRk)=wcsainYCryYzb*6S=NtW+YPq-*HgFA`s^|iHU$lq)ZYMgqCiYG^heU@sr z9(tG6k9^vuW^NbT1v?1WsbzA!eJjykc_ydERz+?Uyo^rHd;&u3zQ|U2x;??ZjcB@I z7LjD%WNQ_%IT*)DvQxIllC8D~%i>D(W^!U|gXv`S3Q^6r$~pu~bUadA%08RHx?C~J zE5JP4#|T8OEIlN)R#WBWsmFqAvOfZ06}8_0I@%`7pNXFG0P>i1Nu(e6MBHRu1sHc_ z*zb~uq$2Xu&bhkz*&{3i#hcb9LTAe3?ZDsTQ&c5b2?{CvmKQ8$%OiK&zKNjHAw?v{ zs&At&m8Y30$U%ENKa1QGIZg$Us!A-=Ywr`hZq6oCoke?Y$OgdgtIoV4O!n)sY3Q1; zpHi$pvew0z=)-6Qb)fXd8j(k4bs-TfSFM?)4YL+YNd@o*%{FhxGVaMuMj*M z8fBl7*_aw<8y{?lB*t>Z2jpQ%qHafQ$_Z+^HBsIlDY8982U174?bKSKM^*#Ax7x_z zl<#E*?Y*sRcZ&s zzb)s9(9VfU>kv)Mw<%Hwc{~sh+N+Dm|A_T!xv(SDMw)~-cFhcLkj_(`&<}W5`Onx{ zHYb9X{4Y4uf@dzW4YWOpl-Tul7W<3b9zLV?18Uknif8@PEl*>NJ%ZLIY_S~scxr|D zlRaHv>31fRV|(y{`is6P&2mgpN0nHWMbzWM2Ep&vM$SQ0kr-#KPpY^hI+ViUnLFo) zXNELEzkUKdi&|!X!!b&--5XqkipWE>A2m6&QaK_nz)MjK3UMrUBI`BrJ@~IMmtN1C zVngH;(cPJpvf2JM8Vz@~JpcE#_a^BEb@P`T;R0TRj zosQ1D-Ox~?@nWJ=rlK~rmk)J_#b;JiVk)Zk%&ZaHEM2z;)z4rH;5VqD*&{7R{$#ys zZ@M#Z&c51~7pX#RlwL&EP_t|!qgm4Km^)NV-83{6mWJD^6^M9wu3|=?M88D_0;**U zlI!_yCoqzqj3g=5OS+VGyP1~m)E{_H>Q~8!P~Xyk^o%S}o>`6L^(t$kyGU!WF*!MA zv=)b32DcN{!8ZDDu{gUJye01@0Ae0E%<(i_0-uPkah9j+1a7m%mgfFh#P`rUGT)vR zrz1``oIS0S0%>X{5w#^! zRx=thY0Tnx=xlITG{yQd`JyWR=ZhY79&J;2*#GgO}3H2hL-=>JTBC9i`>hp~sQ z1Ih?$IFo>2^?$-Qpt;0q`vGeeyUuz_e4r#*`%_)yB6ygVvLusRo&Te+GrfbpSYfD0 zCOtLKcFK#AeOOP&GusJfubdE+&|l&2=riiR3WlzdeGC_nkHqTI2Wn&6M_P_1Q?HQ4 z(s+M2%bl{Ca#6Ve^}$B*U&zfn6C>Hk)mVGSR`RzXpSd{FU5IRHM#cvdEu*NvtPSlc zmZ4JB$ds&|!edem#aTu$y)4U#6Lx#(DYh}PY}aHef$bPNU_Nebnfcv*P&gGelKI#y zaZ98h`p8j-{5>a;XoE*ALv3}fm)${e6?uTjj$D?noBo9EFrysS*nR8&1Z=Q~5K*EZY~-ApeTQWxfdRkZ-dOEdl;E428-|U9hR}4(S_f=56*hh(Gv)>u&o; z{iZC8g2}T;ahJ})3a?D*A)AFE!zFzpFYVL$KRjzFOZbn_d!i1N5V>acv!G=NnUgt1 zn~+sW9Fj#Uim#~}kcXg~>QCNz-tnCHoUxh+M3zV9tJTxmJLXY)BeKm##sg8YJYGY& zQ#2!Nx1`7mrTgJ2_GY2uvGHL~OvbY1hV}}fTH!_bcI>_NlkP|6a9*LvtkdBns=H%Y z@NDrNEPiKC^e+^V_eigzJ;M#KZBdtyW8RcJ6#!SlkN$2Z`6Weo}rv*yH_hhMU-9X-NBBk5#n#F1IH!ccEmL#`x$ zIIo*?ZL5{l!I}6_Lz`f!oJGbLjF)O-SMfp0@A6E;B%}j-oY*EG;@q|t!E||~x?kGh z&%se*fE*?C$U5w#xDs2TYz!YK#^4rT58}G9%J%x;7&%?`Taqo~kvGDk9j8Jh**bJU zt{FDStN3x&7PjeF5s{Dl4jI(0a{&wo?@FWWUryK!>gt}0hL>H%} z4#=PJ55QpmN8V>nwI;xjvJThCT3LqLS(2z$$rU3X_>O#qLK=nCs>=D|LDqhBe@TEb zX*hKR`K)$kUl-eWm5|Af;jflZXeK%YW~h#;!&1jSgdauqQ#{D`P?BxG(XS>_4}tMS z41cQ(k_O<%#C7{)hUJ%_%zCo&bD}_M-04_$G1@-C$c>}m#dZ%5lwhL88 zK1K$Cf+{7Oi-wK$z$WHPfP^gbHW0$-7*lhpA^r`z1^7{ra)(HgTiB0Mb@Z!<&5@2|kyukWEME(qP_7#zh=I%jJ4eRI+o6X1yx;}a z=^h4EHm=YQmR0WuOG7$S$R?7Z`RaISMeva&9T`uS#|J33AS3-oaRnEw@8zp_A3SKC zt&FR+pRlG~A%CeP^`G?zvcS6Bbb#q0l!ix?X%%MSXt2<}C|CvQC2jFfVzQ{)(f-=@ zwi2F48>&5sd014fimn2Ws@+u&>kd^8`oSgAxZp<1N^20PMb{+0sFm4d}0mFb+YnKjbmn}8H2E#8|lD50eVo`q4jd@=tuYrH3#|-K4JglPQ@!* zClXx}hK469O9{f@B}PZ@SxT{m>V7zcPY6H2Yi78}TH&VZ085SH2>#V}nQu&vP!x1% z*g$lZmPwDyA)+9#Is7^}3Y)og3b)Q8sZk4vYyqpPlkrw)AoxXIPWi|^-m$@k+jdyi zQLlqR;iRV>_C}swhK{<*+wzN8mfDYX(LA%ddkxr(4>M+$}X%I9L^8-bQ z5i(f@*rWXL@Df8Wxs|t+Th6p}ZG*SdHL=O&9%7v7gr|qOG4k0HO5JB9{4ePKMp!oR z`H2(h7*NF%V-86#(bMDs=L(!<)|%Y^Cnp_cZ}AE6tjKYUe~>E2ufwWe%;%KR(U z(Y)7JESN>c5$#zJ+9ywRH{q`_SwI!$1@WIBLSE7+x7ysp)Lwc(mk96u2bhEA@4=?J zg>XFHTDT-n$p_dY5($kaP8;&IZP3L?n9Kz4D#f)Nu6s5FR`Qkl;sJL9*PR>jdzrY zVM;iG+8!LnK=5QYLIex>xN$BQYgI)FCD(Zmr*7sQZH_9hDg`ol^ z#C-LA9DajsW?F~)iAVBS@jdf9wG;>;ozMwhGm0_m_%N~5{g!bQR^md)W|5cs2M6(e z_|xOnq};vz@zs{B%RV^J~t9CzydLT{#G@*l`KTvE(Vos=zu#9!bEys>)? z{{j6S?GCoHZsisK3NDFAkUtQaa7X2#Tnq0e?dNEwIq|Re3?`xp27nuQ)O`0jb_KNPe zd;;3aRBT;>c!|5j#=uJbf6Q$bC6FRY+V8m)uxl1!``KyOOkz4al5H$*r$N-G|6Pic zniS_E1L#q^_xqmfD#}LU>55JT-}U>U24Fj6eEueMummVkwg$QhE0Ssl63C_a0M55( zx$lc~L3@M0z%B7rFdC5tflfll?fZZS&<9>a%*AiT3{MN=?}crLDIBN>b` z57;l?r8^KY(=Hx?ucSWUT4cOntgu@*$S@U;=Wkl_GkV~Ov2mv1!J}9Y<5lkgXMN*Y zr44i)>q}N+7o)B96@hsfmx2G$0Pzp}&QO6*+foUrF1J<^vB!p=P$J%x?VSyxM){iQ zvAZ?A6h9!Z^MTUwbdVsG2#*o%g~tJuAD8bft7Sj1PfFup0e9IF&uWl3u`MA1Y4T0u zH)I)_M!;e_|5g62du#PKw2nTg-D9qeEOGtGS0;Y=t24cc9G>w{b#E6(Aa}%Kc%S*E zk<@bVZ2lE^*Xz)=(6%-;GkqaS%#Gxg`rVcx-e2%#(;M_9GCj36+R)fvu7>~MYH9W6 zO70%AoOHw7Q|g0m!Bg-~{?)=_#YZN!-!77gBdfB-&hmhg&tt4j2uNzqB+j9m>yXqn$(O6i-uOY*_rok>Fp-$K$EMX~TK& zccAgsQFZ?3@9Z1mGVwd5FHFT<{k@ZwwR~-F2gJl(5ITylTw}TIq24U%Pr-*^INr~A z*1W)QgU-n6TxW$e-nTtykuQp2vR3RpJT!dCdEVqev( z%sG)N%2S~OF&#+@ObO=k&5(I$GCkM1DDQ*6JNg!D$<|ReDfLTs3NGw7sw39OXYdcn zYz@pu{kj#v0P}ToEnt^e$~}R7f$kxl`8zv3=Ob{JXv{ywmWz*(Qt&N+qa#fH(4pSS z+Ii9xPrQ{UdV2_DF`S3@#}b@H;z;hh`yck9DM)*;3+N5JzqFseEH{$h5U4y=ndZ>} z&)IiCe{m+bpSZ@2DX8Ucq_sGkp%{Vie^{y#U$B+l@yNWSjciw@hGB!aO%50uf(ltE zFURK#L*3KMoP+&x7qWw;sn#rX;o>b`c9YkmUsKQ$x`rRZbBQ|0jP!SF+vo!Y?Q)E4ifWYTHk3+xD(VtT>3Ezy)p(k=WGbWgrq za*gjG)MsjtQvg;h50AuKf{&q)0pQ;9ZB4txD$;#RO=(`{TYkH)5=Y`ez6t9G9>W#< zq_cu}(`0q^Anm{*>@83i*-CT+x*{IoEyn}9k!<#CFpgLrn~%H~jumW7_+-xJLcxK^ zy1-21R&yLvHvschw(S%yL@_f#k3c8-`XsOBFUe0?yLA*gpWVX05cJSfCE46n+=EtG ze;)fS6_aN$pJCK^L%$R0E$1Q+1q0U?T!C1Cje*lpV`d%L9gm~Wg9EUo-f`SGd;qf5 zEgJ@TPnUJAo9ybsJbWbohD&sv6H4XP`W5JvvMw;l0uz0uPwoKU1|1^9^gQvdJV$v- zag0PT&^681q8f#xz=uG;VAJ3(;+ZNKYO`;}J(jyry!;_s ze(H*V$SktY>(M&tTzR(nZ?O{7(EXaPfb2$%8GBM|HTOl_Vw_D3Fs%w`h--Qj>j^9bH;FH?L&zU#^R)+nNa_}TlXjxAT!f)aAPZ(+LB%1jyif(MN@BV|B|hz5?K}$jSLisi!GFR9v}Zxo<__8MFWz z#J!?5>^b3gdLlF!Ye_yfX}6_gQ~1q5MQI4|8r{8PyPTF$RdWOB>ZiSD3#~|Q@NZ%= z6cttrXW3(%D=88AEqH*qhqy$Wvy zE1SO=5ROJl*%!rAk;}vZ^IGdxF&zqGIY?oaXvxTb#>Xr=u1V@&#&md`5%Ul8wUcU7 zHv{Q>v2c?Pu_fi^$Q=q#!X?0O;#ftjPemZs_2!#k8s#@Df1xR+aiRm=2Nst-ukQ3G+KtX6UINc#I~pU-^3{-@=rsBx z*a*tyULZeeFXR12QgkR*RL-jNi>4UZiuM;)mH?psBIhKatp}J5D;w=Pd zufqG$wH#?oH|$7#gY)t%tQF>iH{m8Qx&CeRt#n-IYM3ZaX4)9P8YUx{`vQJce8iiP zL*^c|SE|o!aSH4$&YGsg_|(4**8mL(fiPV0M%w>M>orrFfFaq|C9%Kw+qXWGs26y5A!!=R*)}izJNE`{j!~S&{G3HMz`?vCr3ItO>I5r zzatia)46x}2ebu$9;<3ZBaSec8kbnU=yx;^Izq27e$_ozK11^g-!NIeCHPo*2&7S( zx!Y-%(uc$j3Le~t+zKo+)l5x<*FizEBE>`P_!(TwDv#)6f%e|!ctbvg^!tbK+F)bu zuGHUuSxP_%$>TK1ONs}uL($88D%M=91#fvy21xR*Y-KJ8n-i-6=UlU)F>*ZIj$B|I zrzybM4cVrkubHzqxP!aEAMx)lQ&lu&7usVihm2xX#zD=x#tjcK~{J z2u8|Te06^quEu{h)}h<+6~somSXtShR?r?h!n|Y_CdKjl44t_gE^}Lc@)-J7TGZ4Y z-n*%l5HH*)iV8DLySG~l67|D<{~ATtQML{~J+Mmr*LMTGQmm=%(2rr)7<9-g;FKUq zT5KKBSzhS3d#9!@V(SarSyKN1S?rJFZ^BpjY*^<{M}KSz!JWB3xPj~;1 zu{t`!nJEO*=kh;w59mp74mOPFC2huo;jC&r z#Z=+@2(#Gt5G>MOYv9fn#obY8C>n%+>4Kt#ya2p%-=OX61fUPl-KZLlBGm%honH)3 zeV4p-fD-8ddl?A?Vo-LSRnE`mP4rOkNb#Y-KKcQ8(KpKw<;z2=Za!AXT@rtS4Z(9@ z2jDyupI!lYk5yUwoGD5j2HR*d&|XtggwZcu=lHgCP3}e@%}^P9Z|DUKK^7N}W@j2c z!)cI?aWk8dG~g1`1J$|f5#5=eKn?M=F`-~Tn{U2X&|Qv0_7)YoPoWlizAM-Jj^D?+ zbjLiWkO_t>rUW|IFh|lecW5iPf$a#igD22?@(8{upoP|I>S^F@GP*_6Q+!|?>mdTy zfZ1Z6ueY&6@oOZ**_|eVp{dKcolr+t3AUUqzpWyYp7xXe&pg;W&=@aFhIvgsv@p>q z{O}J!Uvc-Osm!~g7pA+?3g1V1t0};3M;OC0ewe2d_?bSDvXoyLmd9M6Iqk&iYvc>Y|En*L0 zCV$cmn->cmytzOFBjc^RJsWDoH+G*e9|y-7t7v|APcn}+_|4mOn6H63iyi0l`75Rq zhPCupua02?2kGI)Gya`rduz9@LK*s>#O>iNYz@A#VHkgkwg`=cvAVY6Kl;P2d&~z= zhZHkQjGJAFz*I9MWAIDpfzB+wPR&jE=_^kk<&H8f_z1RYYC{lL zk7>a+b^1`ucwQUBz%2&275c+x}E=)`|b%GaY9(#5;Pv@tA!;wPW*s?j6 zKxadaaV%EV+1F4V+(TbO&Y|PE`Nmv!s(ZPwz1Yy(0K$wHHTAp+=-=FHqd!p!%yHf# zR%!lp^`H~o%YgUAz3V*H++*jlO8~2UEAa)pmxyE}@=f@byqa1F&y$urE3xNYBisj$Q7V^x*4yUdjhyv3m#B?cE)JKFX zu`9VvKhK=Y#aem9WhJYhb}pQ@ffOeQucN1QyrKq$j_m;ioYu4z2-w?l2f?M9 zf&61btJEFre2Z6PrFC%4V1d7#5QN77ze%*)G z_JIlRowOTDFg~Umpi}6U@S1HwZY~f^$|mC+U!_F_+X_B$>$&SlL*Wy06}o6R$#
    }?<>9NvBl9BA5|HjSSq3>PLt*<22IQuEpQ(>x!@V2mIo zv@}6{b#|aI&(t`;C<2%RU-F*hpK@!&BJndoA?<+W^lQ{YEH0?z>tPOf)7jeotm3EW z3-mNFSq^|Vgaz8l=mEY*W?6>DFqCiOED--lPoOP$vImf!={C5o1jbZgf+j(E#uKj6v@@D@^p?PX#vN)lj3tH9Z^)mb-n<2#OSWaKYo7zR zlCEMQ=}3AO+}>Q5Gk`iNm@e|8{bMuE;ZEczTgcYf)(~WQoiR>YUc7c&bFsJXCX=Rl zf!sB`w*Ia=X84C5RY3s|CI_j@R)p0J1F^)J|+LYhUGiQq4ZwJT|6hhCeqAj08Rm{VD{cl zPVw#sE<)3o*Wf9%k8T&B(Qc;4`IX}P=`Xm-riEZWkja(sS5tXEfSzGL!JG9Di6V1# zcx&La`H?0YOv3WBD*>YZHnBL2>GI5f00)dW(>q9OK|u2ns%V~VUdGmC^G#U^9SvLb zKIdP%)5Ul68hAO_ncfSx1RI05J>T5@@i_%g`7hArvaVHxEWj#Y5qDGKBDVs4#3p-J zt$%|(gFEY904XF$;jU^>XTvWs9y>_SFr4w`5hie;@e{|13i~_O1ni5kNw4t1iM`=5 zSS~Y`e~28UAM?+_?>?c{Eh)xr!(JA=#MW^`{43!93=?+jC}>4DbG4@c?gUXAdI8+h zpET}bugFn$5Z_$*#O0Ln-d2ITp3VGlHo}bLwIbuH%eH}T^Kstkc#P4(#dVHA73C}D z7259%#C1^aITMiu{(3ZmjbauW*Re5gO>8wgg?~xc3Un6dL*tb8q}zBz*kPJ3Ofqo7 zQKmheltB9wv3=keDu7K4^NF@wS|MxTaljz#DQ)CW`Y@!g#Y2xmS4xkC-wcTJvU8gM zt6@ueApnW9_*$?EcBG|*7+YEFjE+icomyS^lr%|-1N(~sb6w#&GF#l7j+i>ijJu_A z3HTCv3jF2LxW0jhfkUZlwV$fjF_+*1Uy-lPZbqjgmEeXn$y|niW^6aqMxF?b(7I45 zJ4|zv>kaj%)y;e8j<#(6fY2IC#Lgiapfc@n8ns9MkD1`df$!{UdNVf~UYOZK;?>GN z6|O^nH)aCu#EQN&@hZ9p`8!xitcXqwj6t%XQlvMt3Tg~*WQ>JZ=@n3WsgSxw{}di# zlfqdBtEmR)@m0caAieQ^L}PKF_zB45qjiNcC!=TTKz%kCNb@iVatNP;*J zhNVtMNBVktk1vPsz%Pk?0v4O<3R5*kPbGOOvzLzd)dsp3^dnC)GtK8!H``@AMUYW8H%xJ=#v{2`=TnN+#iN!wlp! z`z+AL*&jO1HNh70EP`>n(6f98#FgR&zA*zZE5sKrfO{}Y&?U|T!W3sA)0pp7ylF=( zXQEJ@pSW$lR2i%;qO6i~kLif)Vw!=f@D`|s8qkPu5Mv40jMm6E%|P@L6c9G>ai$C5 zu%QZE4DFTQ($ihpupOLW3}?XT`N$ro74O+w+f&Yy2K^=M;1{DUnb*(*_>K9!vzs>G ze=K95v=wgQzLR>>S$+E_up3h#6cn|AL8Fu15%^8iY0EQ$ewcX0JIzxhWI9v126R(* zd4@o7&68xjxEDSg{w)1;H!XYurd6%N*|pP!qcSI$;i`1)jF({B^uy3SXpGR`o9Eq_ z@seH2Yszr*4y+DXTfWUN_Z$;oZW8h|Bfx)0X0kfpaP)w%$RyG8EF+22V6_y_=9$Q0 zGr(jali-g09{>ma3sqsg=xbeqsp96jhNRe7;Ukwpd}5Z@(QG2j&UlZ2Se7U<&)na$Cku=+;2n)F?el?BmszGJN~h zk6a`%j&Q)a=6Rk3c9V7+yx8*|)oCddr~mOi4$Ly1fyu<#+>E4+=svlZX1~8ZQri;- zM!@}$b+E?!lyzzQh^suq3o^k>c7dl0-=FD}x)lryD!Ll!AvEzBeTk`y1Oi8RS}1gh z;uYXAyBNOY-a;q%Hu;@Ma#>wEPY)5cicd|WjWA%;=NdaWonk#YRGta13>Wv@&?*l zRwXzyTX4(!1W6*6Acu83ASFGk;Eh-x{>F_e7J%Jg3Ug1Ng5G1EAXAxTQWaMn&)D=N z?Q!rtxEZ`f|Hof8bnvzZmLgLHGkA}70j1z*V+mUwE(O za2dEmsEgGH&LQ^`Un>t?Hr*PbwP`$3y6)K}sPpa*>eNfvqJFgyyWpg{sfMh!FDe9gal`)}IAcuk=<67)=Psc;YA zrl$Y4a_m1~Q(u%j9k`Y@Fe${J=8x&Cz!-Onw&-5~lac~RM}WkRX&y*Tu&KgFWVD&g zZ|DCXdYGOIW3F7GmGN^xg7+A++;m|wG8$?EbT`G=SKJ*WiZm!wVD1{5v4er8fu+nw z$io=wTeuYRII* zv(`IM$eY0R0;U$dK?{ja;uw16)+Q`VM$&<&|e(FJrD=DX($SSzJ+ndh_HSDBHdUIp<& z4gYlC)8g^qdVdO};RhN1rT54@zu0#NDwUffN!!CPmRlCfGdD7ILw_1ZBB+tl^cHjZ zhCGNg=DE;p>2EVv#v+QqA%zW5m!Tup4ZKh*AP*!spkoV~+3vzMJu@aTYyB zxWU~wyNQP;1{tBfZLgc(z;y=d%YF-Nhj}FxS_BbkYyH)P#4XSC?-;?b6G_3&@GImt z(4@T9!Wv zW5N`q6>JL(K)q#5bCmc#Z3AoN`k3z#TTD0U0hpv~nRZ8i(SLj6VB)+fH>nCBupuy7A-Myg_>5 z$$u(Z?(WgX$)}Gv(P_O$n*e-huHUJJlJUNL#EbF1^S{7<&)+v(r&C3 zJH`ErZC*H%-^l&L#xolNgXluBKevRLMNdq3C3oaL!imIwW2W#Mod=CG9}hIb^S~T# z8FE@IFK59=O{e7nenNN|n81qW&fAar)})q{eF>}iR&qVyb$UC=kfbl{f*HK!y{%nw zOb5?9UB?Lb;yRo0pJR-H+-QizZ3Cy*S4n- zq+j3^yNk0HJl8nAwE~UNc&=FN0k1VyD(fjGdLo#od*fg18Di{%Or!~KUSW9%VVC%( z1&ZJfSiQWj=4#rm=7rAr^d-+;{ z=tUWK^ol25_pmSz-s(MLysO({8cBC#dejy??Z9HjSG3u6m21W~D0m0_2z2A-!1s+S z3-eu#w3ivRAnL7@ZfCn2Hk>b8xwq~s5CxU1E z&waZMQF<0!$9NOT=Kbb{NJXsJv{sxh)N?1D0RooEy}unX`ntU0u}w|{LPu>QhCFD zB*XL2yUCQe`ww3G^I}74v3oan*mY2o=S=4P zVpT(0ptnm46VAa&9)hqt<`1xmXcMnba_9~+J=nv^tN2;oR^Vuer`MEW zh}!gLeyVPg;jBAPe}G#^Kj9iVFZy>1T44-S%X9~HQF^LvsA48>AEG`M&XQl0DS;I- zt*BzUy%XEgS5NAsybLZ4&oqn*9}4ZUOy8_@CZ|K@D#W_vQxx^-Ra@=c$`& z8z_aaFdyWj!7FGLIi1oAwSmw0zR-QeM#1Jn<&Q`M@%-(1L@|aJ^8V_z0;b0qbbfgNI=y+&-fF~+Bmcv#*+L}Bh z@38(5|EGq?1(|F~f2z=dhZmyXh=#Eu^*ga7IFdXeq#%IGpl-A|HI&|*S!T`#`^mF3 zZ84qDAoH1ZF|bf=EuGT@-~r?wTZ_`L&8zGM=gUCK5`H1~0{@4oq*Pa(a*ePe&xmz) zVYy)8hJ-SYCqxu(-v!%agIHYK1lyYyb)5M3R%^OH{pvQ365naNTPWk^%8}(LxL5E zD#mW)Bk>&ajyx)!i#)M^WqQRj-o@JmCF{u09N`aJ-S7i_YbcM31L~LX*dgmSEErgB+h{E@ zeDk%`lvo|XS7KMn8JSHkmsi*>qiMXwyrULJ_9%g3E~FlVWJ&hB#NWZk;3{G=cg$*2 zYM7HlqsYI-Tw8CdJ>iLMRbBaQtktM05g$??yGC!MEM%*)EVadUk88=#v8}i0)du9G z@a4!2e79pZ=OgE-lY{fz3v@St_R>vw6f^?sAS9rb)IqVsmJjeIpp)f~vj1>@f)4e2 znX;PdusHZoWBFJ3c({RWI@?v50{>;}r98nFV~>$?RE^A)Ljby{pF(g4^Qmwm@)q*Tr3vKtg1ajI>nS zND|x~9^7RYcU@p{ef#}3pECLNbXDDZ?m7SJ!-%dbwo}1Wd)U@ExLLhT5yb}ACOn0U zRQ06=YNLWD_L0PL$7A}0V^ruY-0SE|#Mo{p4s-PgH>UO~Kecb?*Z{A%a!b^0ZKdyy z=3$oEPDSZADiu=J5ryVDDb1Ly6pr7s(oqRL~*sW z-PhYO)VVWokKs#S?x?Bi?ENSY$fi>re^C3vI%0@J0DaYFABPWDDrPOCD<9$Pw z!5lTmnZ#^XF9Sr=|3*SnsR1ltjV1SNe>xsYL}&td$?aC6smGx!1!BrKAZnB30m36@ znSNOCJi#~~Fz1MgOc8v-`~+vhvt3n*+5W%OuPmpm4u6(wX-=k{!9`qayUFWJS)yDH z=AaiMKZH4uvml>xdi~-oY&??nEWKm*BiD%G+J|rh52qOnEUxG;rp+ zVwi8W--qZ><8YMR2yUQ9S{=+JW^E{)oE{d`Gt4nnGkPBCy*-YQHcO9G6_yF-Cdff38r0Y^xnK zkb*7<`=CQzh4NO~VyR$X8i*}6hvcNB{D$HUxB^U0p6b{x8AB_n+2S4I3vnMwFvo~T z!@V4r-Q|dM_|>t-#$##DA7ozWqxu)og8ZUp)^!q9rQLvY|0ekHDhr_FwuNuaeWsb zbCWq6e2a`xEH+1|GW$_lBOB2gRHxELpo>x`Jl1wS)W;q}!a#iJW$7AXx>VKK#knao znw}Gm4&5RS+s3Mbv|D+rJ%ea-ZFz}6I+m2G#!tZ!rC<1yCVgpNxWN4lAK>bwcB1XB zN6PpDRJlRljsB{1w;kM;7s^s!mn=;&2-{Pva)o*|iU)&srL_Z>;vdjmJS^xOc(ysg zcG<97EN|C{8*IgGzbGkrL*hVXr>m{KhGTOE18-1^ZL_>_O&{>HoEfe|=u1?t&(rbaY?aqBnrSMB>x^g6?U0`Z>9MsD_l?EyGlrvI!XA|`y zvng;C<_A!6o_dJAW}F{bqqJ~lQO$!5nRY@WNJ!PBbxK*=5T#e}4AqEZ!#9?>xG zqf+=>cp41QgNpU)O6e_@V;kgo!>siA_=U+Cfhn$h$Eeb#fgMCz$jamfGv&SZEN7A{ z(sjeG2dixjVII843y#9%*w9J1#+K!nBsU2E6_;A#1YCh@=3Bgv3c?j&9_R2 zvy><1AG){2VRd}?BDT%o<-6MkIeM5f0xb&mIRSaram6v%UKym(edQsx&&qZ!cNlbB zVg3xR1IF-GLJ7LXG%^4>vO64QTnE7bvP|GN*$k;{yNlz}{ct6d;w-lRTbkx?1fDsY zZ@Gx)h0}b&a5p(KxIO5y+r;$XB+yPe0$`v>Srm90xFGJ+=~BwDBWxw`+m>a?!D3z= zn`e>`Sq~P19cY@PoO04~HiTIX(p|@<@F-6&=dOAOlpfCULKkOsQ7*n3a+OjhLC6Vw zBrAi3T5fNKeS7$D@Lee2N+?(q=F#bvibQ7tv)8d#3GTDq2$k6Llg5|sSN@a7g$FuL zyKYdQbJdj;rm}s&!+@DeBpjfU^jkPAUn$U`Fs?9Q(tQ@u$pO z>`X}o2dbUAdoYPO0s&@)-pWHwc9y|lpQT3fY%NkS1X6NIj*Sja-;C|fZ29B zJW;uq`owYFlvkQw(p`H7f0M&PX{OawH)XPYA=^`{gR4-3*>}!$)ISPQ>>|@`&0x=f zEBqqmc;YQyR>@Ge;MurZN+aT%a#Kr|IeAuD8(Lj_Buu(Qn;YLPx|4s()x85iAG})8 zB3BtEMQN@jqNb&e;4oROwTsK&yq*Og7hqbv|yloT|sZGonl3ZcC>^LRN2=C?2 zsS<~Mn$D27UP zq$qze9eh)nd0eio6vl=+DMrj^xE>l2_#C#lAD5h_26}rJ8&d9sSfF!m4qwnRmj~6S zp+0gQqH0N8V6HR1xSjBvAH&Td0%#$LyU&3J_`iX8M}%{+uwLq<9&#QOPvjGh-p-}; zPusWB4)(I#R3byE7PC#Il!5kW=Q3_%$__O%=u$?m>4#SipSC3hCX|klY{_?Bu_4LU zK^ju>Hk6V4DHuhz3AYa|FCI$PW~K!WIpuIUsyI}sbWUIlJ3su9c6$%Y_mH`z7qPC2 zn|Uvap=dbOc^8y*o>7mNWCz+=r(us_6=pet!7jV(%x|F>j|o(!Z~E`WbPLr_Ew7$| zLsD#x0)BwGvb0h;ZtEc{#5l1{@ol&qXxTTapL3t5yBce$U$`JS8m3EwsX*v{C=g6y zOrZf90Y)F19vbiZZa0Oe*anxZvGa68NA+!gJ5cqA|5TuDXsH?7%@RflJC*;(@KJ zbqdp0=@o8|H<6|VUO74|OPKadO>Y-vSLmHe!!-X9m?7JB>8>Jry4FFaFk@`LZFixe z zSo&8zI6Y>A{j8&D=tOZ|@l)^{?Bh~HgOzZ? zDTbz&QkkeDurO3USU@#|Uy-Y-mGA8z!u%0BZND5W8|+kU;JTvGLRJWXusT_pZ=IR{ z6yL`*u*9KrfTCUsJCro$CozRt5TG2Ur1Rlk;T5jxN`*il&ktq1%SUdL+Y26OsoseD5-jV?(!5}O@UiqKT|n*2%_i1_ zZjg4hkgDyxt?VL?8PuQ@zM*mAw&yLBTDd*kOXWwXzVj=cN7A7d&VBaf)F$^>XDhP2 z{D(XT`wRqnb3#0q1_){wi9wo+SMJ*{1lu~^2BU#KT#J5PbXEFaK~nfCGFVy8u3)ll z1(_At-l4srgJh4S1;o&xS#xkN0)c{FIEk&2+cG(zX>!<^9F&Ov*(Q*c92sg0Z&zZV z<-I@N9%HMd02QDE9M<5n;zvOHn@TUJCyV8=($wT*p@mK~cv|Du<%UDj=RmVyTf1fm z4YYMkQ`dmLS`YjJ*$^54-Qi*67h*H{4YO%^XPnc8zfe!4^HdzR-(mB`q0P{DuE=1( zwhB}VHgpYil~LC-TSCj68*;9Sne;3$R{cx4sj-*7dHvEmQh8Hv1}?hEYG+er!X^uA7P8U2zF0WCgBl z{2N81=z$vYZ|NOCs1jc4V*LUlk{#+aB&jopItHJI^qjlv_`3T6rQihS-*!3*nkhvOTx zbi6POGQ{ZC18a-#TwBb$;7FI)i8P zvKGMu_)Tjgc*s7U8;y2RwuCOheNuM(Auh;q=w<60G3f8lcY))mOtKkD%dG-EY@NZ1 z(l(*W?(d+C)IKo9lAw7Pw?Z$Q2HJwVOg6BVwlNaK4*CTH#7F#Sb~=&7RuTH++sU`4 z=?H_3B=%6{%x_JzQVr5_;yKk3{(}2eJ9(X_FvKv!(;RLR^2}9`?^Hu`SN4~aV{R2y z3C<&DDCLlu_D95h`6)Ohy0ORHd2PD>4L_vJA_fYYa?Y3Kj9?Nc>9aMTXezaYzDIPl zR*?_+YI6``uGY#)+8GSOwn!LSrORN0K+A2+T-1b+=F0+yJv8kkdvcW|pL2LX70YTi zwhU{Ts-vZ=z*xUgs5@T;nta;6PC3zUj+OXQ_CJXwX25sYP4i;DjK7)KIMf;(Cke5i zDJ@Wk|3G}D66ibBJ{m{L30XkPby4lv#?o(n6*!#w!Zs!b(l0FY$xj}adQ@{JB#0MG zjJ2559LQRSk!Pw&N7};3W$-$BboB^kTBy~Q)?#NLOZ`(lLkfy%p{*Dr7Tb_ z#SPja(b!%4W2TJ10ostE(`(p$U=Yo?tAo)*N7Q|3MM|XEnzk(L&+0#8~3Tr>+KawpDXW-vwSlHE8#)qHJ)e; zpq-(N6flY-jrc0%#mwg_2YwPaCBa6-4#vP-$UPx65vvvc4{L)@)DIo6gHDAwWQEh)bsQS3RQX|eaQ%* zdLnH@*$nNoHK-61j52Tvzb@mtZhv0 zXRwo9Mcg<1=aY>4O_9tcp}nw0I1209rzxEY9lJy01l`@{4Sc0mWBFhq7o{wb9AP`n zZSM#uK zP5FJyc2j@}ysMAXO^s3@PaL3TYdkhO{QjgH3_*SP1Mwer1;2#aX{sjnRn~D^i8Oj5S1q(%lI?vkCc&?(gwd+%Q8*16QQ0OgKW>3AjL@Df~yd{ds- zXkr?_E-N1^($x^&z`fL2X>s&eI7n!{r6t-|>_ym>8z{54vop(($UD34H`dpGw zO_mcT8NcBrNePtA)Lvs(ePX(M|FxU=<;r_(OHl8)CvDU`-w<5JmwSg{b;$$X$|3|8 zJ@J}S&HHL-$+NTYtyO2=M7#=`iR(&3L!SPgZykNdYP7C$bPlb6wZbxDH9X=DFjh4} zU2HPKh(fpUoqUE(S9&uUu|XmpO(B{(s~SYz7?nqr#QeQdm>R-`NWZ{y}bdpc7X%uElhk-8(xsTuSX z6)7OlufkPRCz+>LVp${z4`9clzd#aG7Icy7ku{m4#765bbZTTK+dwu^_e4j0Yodmh z9c`&D<{ooX3MIqN6fd6+%H%DV@~mBp0%8#u$@U^IiLE?-%7v{!u4lWzD%&ZWLS+$y zs6$$(To1HmwP+jJO1vtE&`N%v(8Q-Ihrqc&GBwmTOlryPCob_r_5YbtQi_(J)m1}p zz}Cc5F}1WSHJVJqzBs1i?X;Toy|IC037?Dtf$|j1ttC#2Y35f#XJl>3E~-0GHQu8o zP$$XB{+=L5!I9bOcQOWPM6_`%h$;xFW#yJ>F z6BQxbHqd%ZDGGIj&G13Mg(SkJ9O3Vwz9D8Mw(~`h@qt0Ahp23RK(~@35+NG4bcDC9 z(OG6#$Zobxt)7q0U>r219wE%W+d(%q+tV|_73xB$aa%i&ID3caLEVbM0tyU+z51`h8U z?K@J5o6LPig{PTM$`)5lY&ZQX_M}>X`8>&39VC%d*rmpQ@VZ<|&{=pj;ckLY-opu&>%%4V=VGzGcD-6z;C~gn=Ou&4a7f64RkyFh&F)z@GnFq z&{`cJm7sHJig-<|qi+Rvi(WsE&$o^8mZ$TSO7@;xmP#)RDXJUKGh?*+WlvM4ur02&3RQEpyqAx#h1Lt{RY) z*-9H~44g}Tv&|LfNIFwcS`XGoEp?m-wm~bYFQwJgcCj=4FS1P62|Q*%ft6Wrm5g15 zbHpX%MtXGcn{->c% zrscG7{>|SoDG8PmX&0FGjE$}~-pVAA7g^W*`%!F~wYBi_9I?6OU z$It+cHICsKWEy`>8c^I6k0)r=ZMsPwv@G{=;B{!RfhYQj<-D(o))hyRXDqQY2daZM z1r&E250+~um-BQX+*Sm|@#D--vWa+(WzLFfcNWaAs;C1d3(b!{= zli>>GHh$WUd0L4_@DASBv0I6Y>Sn%)-9Zf(E6hhp@7_aH8Xv-r~~zpratTj@1^5JIj$OjfUnKh&SzboZEmT~lhFDbvY+ z{HHAor6=r1!xOsa<~BwN+zHaDPlc6|0fzIA`R^-@z#MEfv05j@bPw@G7wH}5Q|6mm z&D2=x65J7mCq)b`!Q?m}w>&Q;og zkHG43C#{o_KLXR?-$ikzFW%{LdAw|HG&7jH1s+pFERU3-{Gl8aT7(gnGVygi`>iol zIVoQw6=lYCMP|wMg?>_fGE?6|b}D1@$59(ikC}#B&Y*eO`}w)V5abwMR&7K?7S<%9 zJTX)ym}-e&A8c7raExt4+c8Dl3CEyTOB3rkvaR7y?x?ul^MD@hZN)8%LAmaxx`2@L zCoaH~LJw|-KN-EG5vOWGcR_z&Ia8wkAT_k$8T5NcD%J4$bUfbD z>qOomyAT#FFEnL$U^!R-P)SM5C)h)5<*n)&hCS2j=kIa(!Rx-No}XfkOjUTi^#+!z zv4Rx5nPH*HQ&5}Fg`Hq!_9?6@Pi24lS}5_-ajQ>Q75~^eN_)q%*qvmt2~}<*PsrNr z2Y7>z0jtEf{7C5!v5V z3G-zouVvAD*@gS0Yt&Th0;z}63#+JGO^kr0MPJW2L`m*bin{^PRn{m59dR2M6cA|HBam_S!791H){PLDf@y@+k_hTvw)SRe66(0#~n z;a=7UygAX@T!C94PV{QCec~1|X-jYOLHGqtC8I25s21kORC8}l^%}KM*x^lt<@qXx zThFA zFPt;(LjR7w%okH`GMj#W~) zf2IpIwmdBO!u&?ocuo}DNZsPAfyN*_mCtw5FCqMMx8BNBc`(hkRi%T2LkzE& z^VB#l8ePwgV+Y&bNW1ujo^u62I#v)1pzgJ8s^f~%TQC##bDeaL+|KecYhP1yYK5;n zy#pmYZ8?r>LYk3I%uF@GRO*|j^x`VAc3)8ZC}7l4ZU7>}SCW%iM3kpv$Ui_t?rP-; zc^QXfs5-er zdhu(h#@H34R^Y2_5dZS7z<&{QV>^+56KT@4Dp9aB*q<2g{Yj(BYvNzB9#To;OMI8# z((635G*f6qZPZ9V+3eNag;)l9AY8-XHFu)75)b@;iWXQAY@~LXu2_$o?uxMtqxa@) zCCf>DDHbdFj(yqX+^hv;@xqUJ(aOPX}$@*7$fp);je@KMWAH((>w@%lsPd65YG z2c|jS*r8GezVjVdfbBCF#ebrs*oXcEO?#4v<_gBP$9s~mn1NngN($8{K5@xvGv8-df{H{2}UqgvBzyd48o?T3|_=nJ_X+mxxvmM7mS&FESR z#Ux2t*uBtU(1SWIr=#!uJp%%wS{}-T@3dhw(m_vx6H(-4K;Pg&e`U9adJpO`58+JjJFPeUiOvL^ zAj1*tX*iwMNu%9S^6`R7_NGd8aW~crTPx%0TxFermpYWzVlzWM|1Dtv-Ah~zn#p~# zuSr$CE<^LYbY{IqK|}&o_>Gz(wT24kzEU#Do&anbWmSb#&2D z=_j#)d~16s9Kl;*mr;{&JSPFGBAh~INr%u1OoXC2l1%RfA4sw_WNM4+bvwXC%SbuE zhe_2q+mrw*OK+s9AQ42Vvqg@ns;p&-g;K$2_M061OmZc8*u0e=t4#*y@Oyzxxhy-q zDV7>kGc-!8O%Dl^$nJa!7{XOj>cH+QMMrC?%BiRuA7Rqw@$^L3EYDoj$DYt;8@2qE zl|6JqkR@7C7tmo?Yc?|Or>~FJp>Jb->GyKgeIqIXaHpCVd1`s43R@_FGy9o2gc!W3gdWR$(o+9Jkb3TX<&eo`jH< ztkKL5d0xPYh*}z?wRBV?AU?{UY=4CKr6^>nC(b<>nd=xM_AXpU^p{OVi^1h0cjo1y z+U$dZ7g!Ag=DA9)rmO4v`TsF*pso0F|2t4GXolVCrmPdW#`N*Wc#BMz@l3uMz@%YH zC6ig>nanqVq8e~q@$e$OT!yPn3S5dc)x43Yu7(V~K&)c07>rw(=IUPJB|429FYVU5 zK=FjJx2A={>vz8JMi!+9#!jVlEHE*Bt4aT^Ko*( ze~;a6B&9a`LCO{W1$^SFW^F}}(|?l$;t0M|zJelXEilEdC!I_(d=LI6#)iHk3wf)& z(DsjdMU4SjWH+|9=p{N@e*3G&CsuzA=jo=xH@UCK6UjTMq5M{3e{#J2Zz_UsrB+s} z%2<!FM8>YGm)^Lzrz|o^4F-&w}J(t_*rebM?RDThe>LTmN#dI*ewqKr7*bG)Xk` z8|}{gY5c5QiHa7lQ)Bf@ur>fu8BhoM0&&3v%rAA5dK2G>>$aw5C;D1krTHTZg=Fx- zw9LGK9Eav>Lx%sJi+j>6)ubR=To4In6!B4l|7s{oL+^BJvRP zHE;sPuv_Ul%q01drz4Z5WRXYmE+RF-5ph4ilz2x*nfI`#i`&~)DY3x2<(}$~QrJ5} z6)+r~6?kmw>s@b~$viDwNLC_JLrcUC>D`0pg?FACN`~!u%?i00o5D(pG_znDWrh2x zbS@W8L{=jj!6!J~Qb+4#PxSu1fJ>YSAHK{ibj1VR3$NleAIpre2YQpg9tw9Ouo#TdWy5vq(o3 zq3ztY#5#DtKqExpShJ+wOdN%N(J!DX1s{03h)t=Ta1Hk!j$=j%_0^7IspLMftxV<3eNcxBNO*RxfRNofoBiDas=k ziaSiV;5hdprHSy-_t-;%oWwLt0#C&kzEeKTvRl5wU$b8Y3CvYwm9n1-U`vb(rQ*OS zs!@;vSG^fr1MDfa+xIo9L*6{L5!sB`V9bEEEcek&$){!TUYiAwh{eHu7E4n;doVRL*y8d9>aS6 zL-)tD#VaetLT{-lyAb`s?h&uybxrR;MBsXU4wVypZOu=dNIp_lcw>#IDscOdOG0(( z5^-Mo?camV)3|Iw;AOnj1$LMH0e;7`OdUz=VBb&P!DbLabqX* zGJYX{!FO0WXpKx7$U(6&JB{t?TczxE0Z>@@9bcx-&5MK&$-Sm%csAj@XLw+;xSFb< z#8~E1HQwS#uj)t{%h)98>-R9hFOtPf|T38$LtQ;XO7jRN$G* z_spM4lS(X+=Em`E%6Wy<=D_8J@>qXO?R}ZkRBFuTVUNK*P{(SFLoKM#$KFpeikCHY zc|G6~jq(j<3%f?{NAUDxwZElXMlMxdxe2brisV=E1!)O1G9AaJQ5ty@OrvYcBf0)$ z72kB%cj7qQk~a*;sA09b7T8S2U<_GDDJ5RQa~kESnb}Vt!;DM=a|(Z5>4_gvcbNJK z_tl#*HqNPR_D;3WM?%mPc#bi!U#yY+4hJY@vr9!5c~jgws*SadZ5*{7KS1IdhruD1 zwK~aWR7LqExm&qHJOUq}=9aTG)|?Kk;ZN#-)oC5;DFWpYCp%w$BF_Xl9*OJ7b^*1e zEYm1^EZIeO8(vEIgyF<#<_y+OH(S%d+krRIck_R+w~#|biK`Ou0Hqal*3?}Y)K9keM)yie_|HPZ}A@39=6%NG9Snnlfv8D9GV3D}q@ALjD zXl$MgDiWWA`%PV>VR;AP4P$;>2d!pZqyFXvv5PcIjHbfi8sa7&shu?SUCq6#O9Y7+ zW@=@ws4 zAnx;S;pT~hwF)B4ZWbFjnvesf7%|d*(KjD$3_9swQ-i_PqMqttsy;o5YHD}_-(w{6 zDe#EhpWHu?CUw9DAl0DXdS4?>&lAz?NXI^7C`m>pWG173z+S>LvxARgF)Hl%%jct) zBprgy66>gku%zu*t4 zzwffxgPTJ?w3398T?Q7hE2HakJG48!dU(zTI*Bcq!5YEr9nn^*K=m@$DiLf$YwyMS z5EBBQh4BTI@;}0Dx+0{uQcb8=m`*&>pQKU#v1}Hy8KcrqE$65V9ol4nnU+)ks#f4) z<#NbI-JfWmf+k9<5TTl_^$3(zqgD_z*p}EcPm;UbkAWQ(M> zERD1ZL{OI?p`Y&A%#+sMC;@6}IS2#J5Vdl5lPURCyhX5LQfs9WdfYvH-3`oyl{c92 zBxJvCyhh`Tlg=Dcih*NGKa-N742TL}hgV+vChn>s{K1RU7Q=W)Fi>9!n!C<09{*Q{s zk9!tSNBpZX&E*5iVX4@Ea0dCny;CTnIMz>ZbWbE3E8h#!eEaG7JcLF$P3yp-s9I7v z-zBXpuZBKmcEWk`05V=*LELY)lV{}jgZ+x1G^M6R%1-%?j z#CYbw-M;^@7|_=;HKT%D1CX$6!As)1lB+Z~$5TzozsLgRAP87zphJar#?9C}D%Adwwbh6M7Op1DfAIGY2Z>(QX0cu!F58J&i|T2~zC72#+9hkZOE+;|ctSry|pyZ>TAlV~F!UhqsJRHnkvX@^jf# zFG@`09)WM><%JAgzu-H*nUvX&)GF+LUXW^saHd+z6`_ezSd!o020ybfjENBsV<(dB%#wKy*OA%;7*bq{QngmBg-jy za*?GsH&4FFZ&Cj(`~mkX3#4x3!0ZhF`l7eg1z0ho9v=z*PFmv`4;@m9M z`oL`1gZh&UA)Tldn2F|bLsFFKtx|^m;Ym|$RGGp^bBt7MzE4y}WBD;T-;n{bs+5Oo z)!*(tf+SdY)W4pXsyiTew4T)8gQL8EK(lHCeLZsw-t}DFBxWADSvynY@zTLP?il+n)aqo#p-dkLXB1_)s$kD zOo3pQtQqC~PG$-|e4@8xb8X}_u|~RM^dmj6Y8=7Yx%q}3NDQJEW+1C|=UK1zram;+ zwYH}h`Y!TxT<2rd;%hGh$f6Cz~{3=+>;BCr-@b3b9NUNlFz z2MggxJTh^u??iS6X7e^l&fu?jccPA*iGp6uG#x{t-JgUb@hBCh;<$Oa9V`)s(U>{$ zcfvk#VeTpMi@7>1qe#*vUD*gXw>;cSRFCB_xLZsM#3)r z7;=kH)xH*1r=NIa;VN5?SSc)Kx8ewP)$-N17>-T&s_0B7`99W-J^;f$8_|5nZ+1ca z6}1+(N2}b*Y47I4oF(K7zNwoN?ZPr-iN=Zinm0!H>6z_s;l0RpLuz|kpnXsm8?*0I z!ZE(BQiMIFOZa<848PlSf$fLcjA6qCuXaNNUk@dTg>WB+^UsMNTq*e;t3=NiqP1jl zy}V@`BcaA`Wv629xSiJOTpM;wu`0|HCgv27MV>eQy7~>sL_y>X+0XOan>QC`5H~op zKZV>#y~;VP<&!qx|8en}_x+vzBiEIGPyE96=1w-Z@EWC9^Gnu>?^3eyYgmum2z&}M zJLwFv-!sTl9Z6>^#TOSet=23y0Lr>2`Cl8h<4AN)iy8cn|09_nhXQsLf$d z-dpu=5#!>=!!iDU^-DMBo2QUC(%2tIq)KU=5gpGrq7p61(rWD% zkqUryC&6?WBC@$_bW?GwSOKo^PV@f7+r&}1Cu+`c_wn{l{3P|qnj|K%pI94T9p6D- z^d?gc*e>Kxbcw&VT*j1yaAGOU-NVH%!Rtub@)PK0BaqQlalsmLpZG4f8d*=Hdga98 z)KYRe@so=bOYug|_lOGr^{6r+@TnS&7?a-IiUp1^N~GyDgA3)~mbR&W&t z%m@8d@mJheUweL^uB-ZWVnytRt2|R+c++TkR*{ zs9iX;n-|Kn4PDutLIir*cR}edzswn!b=F+P+@YW=uP4mJMxiBA5q8e6F8?Z*h_4fV zp_lN{x@5}6!Ock3wWp*KxRLwSN{0d~trno@}2T)>cPn#ZoP;7A5rU1=ee6U3qemceoh{S)B1)S9;0AgjGXIcVQbG9;_1fj_+fvZK)|IYJ^2FfS9DYSpXYE! zG}RiZ>U;reG8wsZd<94Y{+;)=P{nkHDwDU&vX`mjd6O4e7?F7q_QN*24|!I|ZSc&Z zE5akoql6!UWBLp6L&)QBwsg%>PpohLf_}mNCF>eQWEuO1$%RWC5BsxA@?~NwU(S=_ zizTLVuPlF<-};WY`w=M-LGL}T7hxe65r6BCBrnGY(HqV8&+~TFNCjR<}g1m4IG}Mzjm{LKp-_FnQ&n(11 zL$RW7xNKxs7E(wZq|C-_eFbf1#1`8$m84C258Wa56Sukk1nRxPO!kNgvm|#uKR@RM zJPd1@Q{pqZQ`TTkBcW@qXj+b)g45$4!+~fl@y@zXoa^0SF2GG#cXWGVU!P8JBSVmG za5Glja}v$>t@P6Q{X!Jgueuj@#7bdD`Wts~sz=J3dlB6=Eps3ki)<#y!f}QYOB3;x z1kr}nemvb*iq6C4iDBa@X|0|`4~qxL4&tYroBR{XN!5liu?Nl7=?;di)^c!k)Equ%k?nVBDiF{P0~uGig6aV*!q;$<^Q2hp`w&o z=Krm4q?@xlO|nXr3SHy`LwR{#VIQ}N8;ErydMSB1ueqGW=VZr%kK$BR6)<#1hFKlhb)m1Vy#KYnFFOn`Z+^-eJQ#N0Xi&z}TVjsuW_oKR{y@4g zdyp!DLep9OI^BkFRd^N+q@wL1KsjW;y#dVB$6LqiJQ37Lu68)r(1HE@WC-H^b$6&)@KV-sfGO z-}C+U#n-r5_L1Dn^EAuFmv`XNmMxexoy@)%oUi~+0x>QSs$+wX&L3-1U0Ki%UjYzhAtk_eN ze`pRk!+5m}unn|nA$=hxY0oS3m@+!u&((syK=wMlOUU#Mv!Ma~cM z?jz)KqfU(w-xV)X1(r*3Ve+uSOV1|1oAemFjr@*wBP;P@QJ|OMh2V-*q1iPz;NcV2 zi|$8TX-gy^F)7IlPhU)v4mY8&tm@OjD`%pFL`x{)R-_MB1QBroHq*gL8iX7LP= zKcPBs!gxDOl)4-XToXNJdJi)PtCr?FvV3<4r`qCLZ;)zp1n-niIzv3myb7|^+UB4A zpSw(MBMC$FeCZ++ zkY0AA$A7GsNn1e_lNlQJn(#wbtVdr)GlLf6ThU?(Q1sp2`=tBs4jW!uiaKpp1# zoJDdbUG3lKIupLCOu(wFy}Cy`#F^ka?k0mYoDxn2#t8R5v@mO#>A`M1BfJ4lQBE)~ zd4AD$5d*{|%r3pdT{pPs^Kr)8nAA@!;}26mNH>MS#L4<{DyHr>)k$p=-=w~2EQBwK zMKXXsE(K0klhwO;ik27eB`?z(Z1d_KigU&J;q1sX%M*u(PNh@PWc;Z8vbrWDYdE|J z76TVL%G2eXP3>#BYgvh`7N#WB(G9Kz+I#wYuEAo2g$|)uCE7yG0>8$aFxeRBEVbvs zXyym*nlgb}pznmyty{<@el4FZne==|n*MX_SMJ}yd9jT=nQ#o*uav`|&0Xr2;T1Ff}({+Os8j9lk!hZ#m8dKpV%S2)ijAHJBE|<~6u@y2m z_*8Nqa$XruSowQkhP_ic6MTr2%QTm&Wy5aGB&TTqHCq}BHMjcE_<=d0{YaUC(wqnm zcoU_u&S_)Z*d3x8Mxdo^wSJKv(UQc@m{`JZM$2Ie%p=x0Jxwj_xNrqDdHT#Vxh1}$ z`X5JJ5VF0Av7~Eu^IvqyL9gHybK%>;eR8&ADVnJz_{S%7gR9V&nz$ls0m)c!Md|R@nD8 zjCYI)uHd$7r-+m82y%JzQe|XJz_7AzZ!!~?$$s)r!xVJ5ISzY68DRUFB;o_~KU{SJ*T#S8_r>jS^6J)2`>*_Az7+`|B<{3h6_#TIs1C9iz|?B zD}&VYjv>K6tewsdG+8{!enom!UOis7z*osf#2rDO7&8>L;$S{PKNg?nOyt|>&zNoT zT%b>QQ9Y|xvKx?s6f+o$ zR^{Fq{taFXrE`VGKCCbjCl(@PkCG0_ucFJOQ)G*h8J;LlmP(bm zaaDSdyTQ20wz01?zNY2ZUv-6Pxrh^9CnVKZSF z+yx@xJzE@3fp5X5M6SQMZUa7BiII!(@2S%GHK20OF}ch?Wp&J2DyJq8QOyOcH~ukU zRQASbsurqRRz)%P(BJ92!sgTul~Rxv$ETzzCZyW;m^im-tHAC*O(=dUpLx%q6L z5Ebb4r!z`ii|rlq8rbPuMYh`a)({f9q mxX%w5p126Jz|S;w8mIPbiU&!7Zr^Ry&*U>K&f3Mm0rCH|3lOvb literal 0 HcmV?d00001 diff --git a/wasted/disco_test/2.npy b/wasted/disco_test/2.npy new file mode 100644 index 0000000000000000000000000000000000000000..0eb85522cbc876fcdd4ca675cd99454738cef653 GIT binary patch literal 35968 zcmbSyWqTA^*EQ}EAPJH7E;&_os=E^;xVyW1aCe8nT>>OT+p0@WS$8KPxMYxF@PWbI zW#Aqd+JNRcs&(zX*4k_D=+?JukAWpi%q_8q96e@M>P)hcg={i5nS>T{ z?6jG)XO5gYeA>*>WB%Xsb|WXx8WVXwYy8ORVX#EdL)@rIzX; z#4<1WXnYT<4;C3ysO_+{%7`_^zwCMXWjZ&u8`X2mE`FLi3Gc^Twp0vJ{;R=b;^&wu zj^$EkoJFfRzpbovgZjj#B=;m?&L!q_uu!=yIjMGPWBxB=PkVpm4|4@#JbziJsa+LQ z#7fu}?-{&tt}b3+A5higilBX{oc>I0PFuy})<`Lk5Mg^Qdk@FJeg75 z%2wMPzP-W;`D`AjEc&pwfdbD2ltnzpM~i9T3yy^@TP~B|f}P2G_&feM@mLw0uvuwZ z_l(s|f6$u~we?qUg>aEK`|a94+UcCk-~+L`Z4ouV<8mLYGoR=pB<63>=76?Yr==3u zHaLr&MEnfIYi_ElBO3jsJyn)lw&B-Apx*KysDDGGz>1`N;V|qaZF3o56y&Hi!nVLJ z=3@3FqK-0*JfU9m2@It6YIg}2e5%bQ9tL)T0ZM6jleh{l;*i@`-6~SGac0D`TwnV zk{*Hj@s+jh{E@t>j{A<^#kIjTWD`>etCi3ibAYc*XVllV*|t=xqa|hiiGQc$3>KyP zOHwtZ+uLv8V`96S&c*as>*jRu-Hx2EuH&q%k~(vVd?h;YAJhmsbN2S1NpBwo`-YCo%&a1OF9EVWSk zDRa`mq>-v8Is^TZi{p35l@+MxdRj>v9N6Y;r3g*ZU$Qs#iP{E7HmKl@;cGK}$x`T* zyvm&GZp6(aZsqoiVAU&cD`J=1$pmpQvxeP(Rn$8aK6k9Q9pCaI`&F)0V@$bv;?Z8Fu>}^C(G|tuTs)5L1r!AJ&e@d+7?X{cQHTZf!tN!)N8QreXYJsA_f<4QAw%!RUj?0G|3?3^`>Aw8H?=$; z#n(iWP_ipa7(p0_si2Z9(5);@@Eo-nSc^7Vdq~SnDS=h~f!Hf*5%v!eO{BxqY=-3o zeI6^P{EDi~pEu?5ZNwgYB82F0Q3Yb-Ucs2Wy|*(cbYV9I%F2zLXV|t{4qlkkTsa<} zpVfjLuNDcTU{AG>xhQ-#lnfn%mr1Lr@oAQg`PcbFWDvbKZ^FJfI)VzIy``zJ)761~ zn_pqG%KS-Q<(H#*z<_6wuUvVX*bQ?u&3Dc2XRMZK8KbBTjty8kUC&vHyBg)j8WRbUM9e|8%F5SU@@pll&XQv-$@E~W zQ_0k>&}Ghwcv6D#2Ujkp5 z*^J$r_>7|8MM(qQ+YOq7V$~yW^@v4%$vpko0NilOs1=Xbio*p0d&zFuWkq=u$xpR z=9WH4;^a2=pZFs(@w`*z~)A_1U#daRN9Zu#-gNUIxs zo)Asepq3bo_*1M6LA=s(w5=HwFZrD!G z;|V5{>uc|4d855UEwxm3CBBU=NbVJU%l9@{(E{iv`3nEZ{(vjB_Q7^qB43Wml$)!y zsF{457z#@f-V2Gw%f3_0W+g*=m0yEu&&8S#z{a?bDk4kMp4@fvHfvDp#l=J)+IKRe z1lkQXY9-acZm}=*u4a&&;CUBIv_Z>djwVB`o4pTsOGrpDC8Gz#cfo{pj(;t`6J7@D1`-oFAuY$m#tDO| zHOwp42f*9O(-_oS7a2Un=e znIfu#+*jXesR7>S942alwT=(g$?kPB^2f*nKuvI8Zt46)WbwZ|$K-2dwr4GK5cMRQ zFfHldwQ=}l^MCYZHzl<*2C^Z-u|HU6;3(#8J#GXv;N`? zyC!M-j6z-`e;fJ>xEX$Lc*~aejiDxBm)L2j8&}(qAsk}|2u-bj1q%~j`uq8ZDj9Gx znhs5HHunM^A&TU7ViDPj-$j=LtTMqi6Xc+I+CxKq^Hu#ab(dZU%4R(x56Kg)JN?L! zt$r3)Ip)hZX@7tgyE8NLx@s8_WE}_TzS2^ZRMj;ds}?9GIJH!GJUpOJg%h)JBZNz)~R&JX}?5R~HW2t3W zV5e~`E8vrb?IhvbCi>WC$}dz^`L2uF?P9teq`bBV5tuGhycf_;yQFtqCa_wHqIxJTBrjam!-wtk2EXg`pQ(fg@YnY_1;@u1VkEV=4iZ(G~@F z$wQl zorf)!qjMeX0Fj3S6o@hQJT%yPOI{&|9evVoXa}e`_8Z%jEU9<(w;^Uzjn>u_-i2^w zq)yJ7Q`~n` z592VVIkwN&2_NQ9Mw~5RtLSp5MpT~8qW4LE(6zyI@hMtlE*mh)Eg@%_AoJQO949WB zW5l{UHkpRgCDoC>CzQZXkpFQn<|d%gdE>MWHqx>JI!(WY8(Va&4!>J{NL(SxdU6BZ zL3#0+Z1k559+3#VBvlPQksz1`8)FGPu)h|&=h1K$^E`1Fd4M_s>bhf7$|z50zrSmG z8_z*@qvCW2^$WyI|4Kd>``q_TJ!Ko9#@J+JhP9Pqy()8CohB7Y_rX)EU#r6WB(vDt z+68u!wt*g`eX@6fccgo8zT8zh6dDaBWjZwmpG9vo3-UZLMVbLB+7Ecn0+5qIcaH4` z%LqWvbB`xxdd`OvP$5!r0`c@~-y!-|@HlMYUe3CyjYSr{w7SH8&`X9#Xas7&4WtIj zQD#yuELa7~5l=1SZBH#ls27-s9*JdL?Z9%doWHBaq&cZ8+C*`kwt!9)KZ(c=$?kG2 zQ-jXToC^NN|0?Y6S-0Z`xM*+Vx@p#|UQgE@v5w!YDePSyNDHy{=$NNp_J2}&LxR== zt4a`zA?XKv`-!i&IdaN_pl278Z>jQ zP`)S!$&*|1#7({jxCHKnD+@i%&xp>Mt3z#F8|bZahv4hLIH#<=b&sQK;*Ise+G@u! zw3isMy;z&*cI+~fU%4toH6@eU!w_Z@p z+7+^>&2sjcbCn34L9e7LbHDpzuzPqJn~yw5f1uK-lcpweYkDv0 z=r-imKI-?Gwj&K#IxOAz;~sg^eC623N`=PfTv*bw(g z#~Q(%Th_l@=qo%`f8b{fW0XSi8}*9%sLZm|#Mf_mq@H3XN~6X8aGms14Y*^mamprC zKcLGm6dgZ|@EXy{7N06kZh;5!K;G+}sMX`Af+O}m)Sq-I z^De-vSG0h;j^I_cn4s7U-sWe4=5j}ETl`G4MvqoI_^ZNCC>t!0`qSst{#dcNP=1s+ zLYm9(p{7c?C`((Vn6W>&m-c3sSNJ(`COtzr2R@32nR@Kp_y)d#wuf?4x{s1(orOj- ztMgjxBx(Y_?%Wh#pB(MIAr`~;UeVY?n2rARH51y?Ljuj+U*)nmsiZ0e#M_eJ=wVb> z>KWY({OwI8en>%byS9kL!6Iu3zm@tog7_8eTxFG^&>mLvf?dhC#%we@a5CYw?T5J+ zbwq&DXY5~Lh9iwC!_DKeq&7;CeI!hhPqW9VpSD5ZC7OV~s~0GW3K5AQhMsHd#I*IU zu=2RX=YlbgdP*^~6Tb;&=dSV$vVN9N;t7O*3l%-lL zR_iVI9;q)`!)(C*V}{%7^EZf0VX=D)HiR4?jbq}Jmedd<<|xEgsUxL_STSW(afYXGi?eul~xrjo#sQynb1?!_F-f@8+)+zioVm!q$2E3tmJNt^y;F`k6 zCVf{2>oXmF?emOt9Fv1D)c#aD9J|BBpY=TE`x$nL148}Scsex$1pC8kaymDM%!f}5 zZ>2p-2HFU^Xo>80QU_N_L~N&qr2JT%@{8Smp-E;Z99$_3;_?1o~~X4S_)X|b|@ zqw5xbBma8sE~XFkMtpLlZZ2ZCM9nN1N~iOcLcO&k?&|hAp)0H|+(P5ncSNDHJ-Cj& z27lBlM&XSKplWyq8YF*}g5pENFUNbm6r}|pU_Tid1RT>$??DA>gR?0*E0sd82}}42 ziuh{ID%@@z*PAOF@(=CIAlLA!)FJVlUXS?XIG=Q1s$d^mu%_PaqE~Q^{TjL}7Y7Qo zbmfh5hV1PE?5*5Ibq&sQY6;gmo-uBP7g&y(cLX+^+ zq&dR=NcT(yPM)G3*6{1qousfG)fX>_^McW6wEr94IzWd@D2c>*t08eLCBiDfHp&-a z4SiJJpc&zEd1)|*7=*c*Q?A$QFvnk@yqvCn@ZPs|(Qd$()D)$cmI4Q1!+>H)l`_$E z;iV&1E*pI(i>4PRwWaN*rc4?3&!PoFKWU!wjGrj=Wj|8~Jhj9z-T<3PNUjR`zvz)5 zEolxu%h)+kSy)cb(Mi~!I%<8O7i0G56F@Q7R;uV0xof^=oz3oG9>7+LU5#SbxGG6w zl+i^u7Onk^4wF@TIqMe}%yXM0 zV9Gi6>aWuufHTTekR*2ncJM1FLz|`0WV&t%riDCk0sTlF4KlRz!Xvm+SS%(cl}5E0 z2LF^)!uHxwRxDJ$!CtOw)E$=9zaTtkVSXQ_7DTrFYz2Ii^4)*Ju}*0q{!YJhd{quO zWE5n3Km|+Vepmlif5JxE93q7%(w!hd`-p81exbJ->F^Q0rPzmkj`D%;iMq@hWx+0|$jHC-D^Wrt^mM~d~pF|M81)HNp@=DVmH_ocg%NDp5! z>7X+&>7_auj}E`L%=G*Z6$cZQ0|XMfhpI6b4oBz=dM z%_SUzoExDjFCjP%P6gc=0v*qPpdV2>xhe%PSSx&D+he5@^#|?J2dd*$Rw<43VYax= zLso~L!`K-9e^f&F5PXH#4_?TRLO+XMvDM)9(1#>sX{33bHGM-u6X7Q<-!-AyeaATM z1?bED63f7O#?JxC@k}ljYs8OXUU?juCt*>&0qkki(T>7i{+fPN+Y~G(tqeC5RPny^ z4ZDMV?>tY>)t!pVmFqZM$iqe6#()d71J$+PVN+MEW2(g#{>rAq()wKaW^yrj23pA8 zYKyjRu~rO`zTx^fdaJhCeKfQ}E)u$i9r`WG4GOjS^a%TL(O^Hz7ctw_Nl;{3;0x(- zvd-qX!jwTgM|~kXs-@Ykbbq3Teh?PKE!N`*XF;se3+qNzgj1CS|Dn)oR0P|z=k&^O zcHS(M8RPIApsmo!=Ceb&A-3M`A0Ucyf#$Auj+G!pb!I!@)to2TuC7{1!SEaVZ%AWB zewo8hSEB2piNU@xHQ`B1xxg*|5y#28C-+RrnIYBo?yu#5li1hlSaol~2>3&2NH&EJ z^owXLTqzx7OV%mJ{O+G2*3q`+U7`B!TF#c@i^Y%mIq0Qykg3gjvdW+y&xISiK!QjSVMuh1`xR^jWol{zD(YRJAT3y6#=c zr-=37-}(e+t6*vD7~WKC2AZ;C_0o{oMra2E40coZYCD`;>eLRV0wa2k9#U&jd&jpu z>2ZK&v@a+}A(UV4B6SMcMfu3Rmgfo`&~SM#ny1|Kx0VkvH?6OIR^h##Mjvm5_v=d1P-PNbot~zBp>LCTaR2thdVNKP6{!Ev zVKpk!Vc$X_ct$qtU8#>Yuf;aQ=HYmyQBVmj7K5sX{ba3_-wI6%_JB)l?}EQH7xpMz zDg0M(KQ+V;jj!~J)>_(YI@h-^lmf3Zdp5FgLx|5g#IEA<^){p~3|7X&dyonp_b&_0 zl9p>@n5)zq>sh&;?sUZ{{hXliK9s9}pmzthm?rbIsOL`uYaESWAN3#hFwvO*XrIPj z2RiJ-Ei;%L>Gm!9EItO#BuSzxz73?(Q>YoBmlCI}u(VJbF~>CyB{BPjKLYzuRW0D! zi7H|L%ifN@8M-8Il&5K@)mo?*tdV?LJ(t%5Owo@yduWZ=J=9gADcgo66kL0nvcz7Q z6rI)C#rVIG2}}|Grx>qIcHdIUxeg_*4jW*KT#)~cCoNMPjk#zrjC$@k=_=AzI+yVw zb`u?xZnKYrTWsUy^zaY8_~3)^J$eY;NS>|Xu5I?O=oY;~eo6J>-s>mW(_wyRpZs{o zH|VBHS$2ST^3+hnq?*oR`zPoQeq>#{%E>07yZpmh2+wBJB`LKE@ge7=bF}TPoD($U z-(tf#Mr~W&@#=SSNoXDOjr)^G%1sTLiL*dQ3$;N|eZyQ`hJXO`}ULcEl1G+mY%Q1FJH3F?RQ41KN9uA=a8 z;$Ym5fR9pA0|EML!Xa(Hm4JUO6ZpG}C0)sQNV$jY zXWL^9!YAtn!nLU{(9YoYub!HEYu9*Xw;ITQ?m5a86W`r6Lr0yS{8ic<-KLS&92C$Z z%mnObqmsMOo!NBqBmY=FB6IW%A*B9m&J#8$;$JR@~+4%CLCWW}Y; zqYhiwIz8c1uo9N9dDLd1kB0I3O?)w$@4UtrQqeHXltRs-TL|azi`pV~jOlay$Ow*B zvm<;N)H1OWGdjGI_)p)bk0sNz{^2=VCh>`W9!LdarDaTiI$b*@PL}fwXHXRE4$q3E zU}x%B7^93EUsR|Qa)>J9%O>g-z^c_Sn8_ETp-ivC2~f?c=op@^6iht=BC)qvRI zOwC(fSjJIM*QPx{ZtW}okGzMAjeW}<(K5HdB_o#yMj1;(n;>h;iqFpn1orvM;N;fLuWD#6ZSXA|l6 zuFP;pcmA1cyfaAmprY8%j3a%B^h)YSIOS+{-Y$=KfukMwgD9I^4leeUcXXEOP$xCf z|B_!2e5nk=uHxgk+2Ok|ma4(lpjz3!F{|W-_D%4TwvPUb_+-5%Pcyfn#>nlF!~QFI zTW}RRk}vzR*jhPHwQlgG)|u^Ae^vDN0<*dVES7-tuN;_T4VT8v%5HiEI~5zw+)BEZ zkmAe>bav8)aeOIqzeBXeX;1aW&NTEl{Wx@3Zn1Y6JR?q0>j;;EZ&A61TiJfD`CycT z3mp}_j-|mL_JpkZ^es4<=@IOtEN5>y5Gv`2ac+l;i5W~R`=HUrI=AgJopRH6Bbwuq`%a2wmItW0Xg-qN~i+tof!O_@^=i@j4vsjFa1sIo|D zf-(cyo6gA=ZD`tbe*-B)A5WEXyoC)Md(jDJWD={#=`q3i;&^={ImB24{fyh`I1rj0 zXOp5cUuHDX_oE$ZLj7iX8GRxx$}gm=xFd9o`7g-GH|+vlQR}5){sTKK@H3#3;=sVZ5F+@JxNisf)Rd?Xx*^z(JQDw`@&NAMoV z=d2(3tuNIf`y#w54+*b@$B5?ntC>#umzj9lCfpC@vdhKH@So^OaEjp&AsNz*b#k+u7qzo_Leh2i9J5Zl4DSa&SgL~S$}iZ_JJ@iBVWpq` zC0dhkno?INpFa|bY?@Y4P70RhJ2(@qrK|QLduSu-9E`b-J=0k2_n?1goM}{2Cv{>{ zvfsh{Wad*LttzNzZp^mk1$Za!9ZAE>_+RLrY7?x2Q=1+bq$Sz=Q9FQZ#f^dWu7Bin2Bf{=f`R#ch(NJ zg5_wPAI@J&^hzrnn-x14>*$iONP8j2@*Snx)LZL9^jlz8UZ#`=(?Cn<6je`u7)+(& z^sdTY>XH0E^Z8(d>@)u7*{n#Y=TLY1PsW7x242`->|&o}IpT;ohlZZ=T;3G_To(ve z+$Y~tjgyAJXGU903U*ukg^p-ZT&lKF6v*GmVzCqP1z*nW<0UcKcSt)oPZ-tI| zzHn7k-8Mx(Lyi>Mf}1;X{eHZP-X^f$bdP9iU&6K_W{~^rsTgl8Abxmi%Vpt9TDJGr zhBMRbt5F9=^8Kv-fdjluG?cGNDd?)LrDiqq{KxX_w(b&qqC zR8rsSenTW|jbm%_56C4_sX&74mZt@m%dPY{I!RlzbDi*;QASq0c;~XnH1j<5Q0ymf zwryaRqXWSsEaDfdEyO`6j@S>H1^|^4ddp1~l9fvYMxV?(6_-XTG~`2)^J-^AW{v7-{c`dgrZ_V>2xDV|q7r z8s*>3%-Po`+mo}$@8`U93#0F+Ew=-Oq5y+pOM&{EmGE* z1a+9dA-gIN@Nb5-94)+U;1sd1P)B`@tuDU|Rgw0yv-o%F0;!>}HKMa8qo({uy{Gqy zP)cr3{qptDx~Q}CPX^NbH~RoP9H|mrX~Y zN~Mn33&`E7&pOxKk=YimPUR?@E$h)ADrR~}=HXv~PhrJXT88_7RF?6G?6g^lVbNI{ z7IfAwudU{5d;UVpY)#lZd~ZTj?CytJcXch>&sN0Xc>mA`sWsSYGKwa?4Y`SB{Dd%! z*yH9B({g^1*>H`h>EqyQlotqRE+R{T`l1<6HG^fuJl2G|NlRcjwQWa7643vX^OV<#3X`5QpT6i%QWj7?>n^+(?-mCee1WZU zIq)BMiOjHoo#h>;%{B8l`Ks4Ix4E6wu+ZiAuRt?^6V3OQ7aGgYKw?6zhyyYYg>e-$_Wy^?x6NmsGc~EL*l$3u z@=NS5e5Dq#=jg?rww5FOFJhjlpKu7BP8`PWPnLvOOI_km@jotwKFN-tst{@RFH$ux z>7K0>!6un6*Z-lmL@x-9Zm(P?AsC1owApa6tjf9KV&#(L;`~_O+Ie&(%_}VxaZef% z9c&_*EV!ke*homhN*hzM*YcZ)9)T|I-dKO!Wt*Wiqwny=#9(2zy*br_jKhJj0d2Pp zRa#iP>Yc$51mWL4^M-Q#Mfrb*q5Pio0bUjBOYh_RT0?uc#E-+LV|Q|2Dt&Fsv0(`Z znF{fH(GaPccIuDC2Fd@DYX_m&V1CoaAMta z)M&9bzlvCC-%Al7fVUv;pm_L-J|?f={M1-y3vey(6eyscN~>rhzC1C}{WmI&ol_Q( znEKqfoxPB0MiYHaG^b%*-bSK+#L1uGIYe*w)eM^8ca#lvc?8{0uAz;`V~~}PB1?k% z^nUS*@QMP`2dWc!4Z8ph;U3aTTJx1=+Gli?``{O}rM{0+h_a=hloRz@nRA3ejA-ek z{)7Ln$2iMl_dqMVkvC)eJe3kw5h^`Zn~6Wi`f>{?z z-^)5z^Q#Y-|9nHSe`>Z+F-sY$X5>WQ3@%&}QP3EvWRaK`yt*1E(MM;CGGD;Ctp?Q zB{_0~)K0uD&wG8!>+M!_wCW=MUwdk9c_d;r5cLc;xjkd=i|Ev`)2fX)JfbPe}ZokcxK8c3W9Uh_K-yq5u@$`3lw0vhlcsK8V;k5 zxzVt#;*suKPmvv%BUGxmmKq>-=Z0Hqk|o`}PRqBjl*POy>+lWN$-Vq!Zvda10SjS{(49kqm;h< zX8KLu6zf**GvAS!f@)fpNo7Gd+is95UIy2&_2f14W#1!lG&>c}ji|3)z8ho*%Mx(U|+HOLMKHt^$TT6m8sFl zMl=j7at7rJP1hG#J1S?HVrU}&p^M?zNR8;4`<@$zTI3$AaUrXuHiR{YkMrx8+^gpFZh3Z9i1yNfw32nGVqH zh{P<$_CD`DH^u#*_G$M#CJp>Dyrr)4m&i5rIpqM>N%8vPQ3JjVHv1_ln;%NPP#zg* zY<`66IqY9?Gu>;?F+EPGa&Ustg`Koa5F7Zh>`&4T&Ovun^4x30w%Wh=xWHszhOdfS zAsDJ04n3?GpWcO9g>sd4U|5919T7MQlN%(=K)tM6kr`bL1gvq&OX_zdl24*`Q^%Mi z;*t2F@`}8(RE8J`s9H(wg;tAZxNb0$RFwDTd&V(_o3-EJ>GI#4$7p~xv{vK>1JHJX z@BXsde%p9(L&!+j#n(lfti$6D%XRcP?iPE3?%~^S*oD^??x5-TVEQ4mh8!WqY&6P; zxjxiQ>6G<9!y|5QQDfT;ahlYb+-ctkUXf$H-$WYJ&(iQtrbehWH%@sdOfWa|&*ulB z$@qId?!B&`~yGBx!$_f4(`=aku_J3!wlj&_>Xe~mIDPsUg|1y(i|Y(pkKaL)Ln(5qOB#U zLE(|LHe|NcIuPU1%7E2ootojcGUtsoeNiD@E!sT6*IKDa(V1ua7#@p8484X^st)w}4)xIISG0~4%L%7g# zY^!B}{!pQ<_W}d3Q&=sT^i8#|7Rt*L=`6XU^iw-2NL0tH+Y$>tlZ(aaT3P!dYI|ZE zM}pkj|G~JADl3{`CAk@O0-qnEedUFUzFFvw#pRzt{s|`M-BttWJ@-VOL!7nD0$>K>WD zo`B5)@6dkwRjoWF-f)%)pw-x8{iKj@O)^}z^s~HWrlZ$bL*~0&pUM$0SbKOEXvWBH zL^A)?Hiwq5Kte2X>sMk#*wbIto*>>9V@z8cyks6kw9E-HiCIPet)2q4)Kq%D%u?Cz zZPZkhXj^XCq-{pWY=@^R?rWLrC_dj()0tIBR7%Tuqe5%QrX&Hc#$=N&W@yPEC#tJSj8 zvBMN3%hJ8`YVlFROnxnH2jA-B+Fa)E_{#qM!hRS=H^5j!Y5pb8qCUAw+#Bp@_@QzM z464=|j^I;*2Bn_(QtEH_I`XM}tUYL^WhnzIWV!ABVJ0DP)%MPEmHZ`@;Tq|3!Vs!C z@Mev5&c(m^#%7Pl4>8f?RAR9|9jwQ3ri}25pYJbEFSKtp%nQ82%L#`;HunbIw705g zB8Ka(z(&CxxCb`wox?rL?r%&4zEE>|F4tWdm8Ytu@K?fES3Ui%^uc>hpN0qI1oLWg z3)aWJ+|Y`zo!wNxsqykJj~mT|-^t2YCt*2A#Oo4_x3ujya9OMd1}WM6)ZB9<1qWlg z`-k@s)0G|L*p4;EuG^BhZKx6`k8kpB16$H!n6_DsgKPMD{$%-q-A{jjn=_x}Rss&Z zzIB;!)_>OSwHAOYmgjk!4GMPM+f7}QamCZzzFWSh|IeM6!);>dmwZF-W9C?9yWo4Q zYPyeT<=vey)A7r^OxYG(4OaOE3rEFie5-_pa$8N*R^pAoUTz+_*7i9+BYG=WI`$!& z3pyDZs7p`=TuxZhSL>uD)3i|DY;h8wh;GtjbcRl=-^DS81&RB#iEw`GK4@0U2Nb2Xau9Wp&jqf6#e6MPAYG3zC95(-9E@Ke z|B5J$C+u2g?2hqZH#%-vx`QT=Hb{z&uc7p(6FlXu1EB#dWk*MtO>87hzf5{Zt&e0) zB<2N`YHx{rhAhHk?gUx@qqY|Rfi=}e=w+>9Bq_CkfLy%MrX18hutvqSH_hG=!fZ&gE)~HIzyqA@>;yS(kyDloL4k z5r&P*cWX8o6Mr$HcNa4$=>zy4G~t_(*8#Ukm0@{4oh}y23Vjk2>fYhMOWS30;2-|9 zxs$ao&ZA5dFFCP2=s){^M22_(k0zU9FVGvV3;1E)$~VFm$D7Tos0IECcr&RIF-3lA zzX=|oThu@%&KlN!1fqOEpfQ(cA-`8xDC+Jmw9$9f@I7I_$?tvPzQeiXCH#8s4z`_m zAXV+ zfIoe;h-dgjFdZ@pW&8(y5sw?MZ+XMFww>o1P{3bLX(LbYLZJ9k-7JU1OTq--Szius zuw22W5$nAJglFj#=|1{z=1%5Sd?HPI!iCS`-wz* z3-cy$MJY{87RvF>g*9lC`IpHcmlof6KZ!EGk^d_z5w$Y?Ko9s>yU}u;$P!y7F5&N` zPmJ_+pEMPJ%U!YxV5Hq`y-tqg-^#3^nx%rRteqx*qj+0K?qJ+1;fjs)m8Xk+2XiK( zZbGVOre$8a-nNWHU&J$@`Ao|q<89#!_Jk+-_Cy^1m9u1};gvufmg4>Cxkn5)KE>|_ zRB5G9BKBkr79T?Zbd88)j01Cd%wsAy+Uw>TxvPWu*=ywr-VE8BcMoePkwGw{@eZ!zlEx!)erG z6Rrvy?0bmTfvL7Rz8bQGrdh`Ot^70F5#oi=T^S?fP%BK&`9JX1%J7{-xvK6`#)=*{ zxFE$cDEouoZ>dJaOO*0I>wRIpziZB{*uBQryzcDgx}@F6UmQ-S z@^zXm4F>3&ygA`v!65H&RP}P&Zw2xB7ku7e6kb`B34-Q|cV#o~?G5yj>tS5fneE~5$Xta{e5moDe8 zgL)=(PdcMyFy(gjPb%x`TrkJ59HxYO*`4OzaF(kQ8UxE{$n#JBntF}3SG+gBr?^br z7Mv`14Oz@@lRD%N(dN*8%;k;>El!RF<@LMKyb@#CuhetZ0jpSh(jL7YxWS@=?rvv( zAL!Qp(0I#6B*M}&v52Wu22jjQuW9IHHp>B$4*2ZeK(McNpLUmlhWj8h`J zOH(1moP(Q_qa)jnZJcG2QsFD94U8tRK+af~AG)fb1?>9~!lzKf5Ysci|t&uf;FS zb;l_-Gr6>DuVuKaLQyALa=NwZ$w}C9P&?^t;a=rcftXy3C?{m^J6=R`Dn07lf!4aM z<*ssXFo2Fv&T~HVYy^2>$(0Uqy{BW4Z7uW7ao>x3UHN(J!MK&-*Kk|X8}drwI5n!? z@}$d=eXDNyF~VQYRnV2sfHIM)&WGD2&B}bgtEB6!GLC&5@f|wLlkRw7dtqth|&;huLh}l?O(GF=neW z%XJslqvt2rMVIBCX?xYPyA-;jFH77)c1r3d9Y$qgwXiEcTkI)hax)`&`;Crxcn0I}zPCRTnC**56pIKrf13xyM*9*)!0Hm%6I zHdxJWkiWCn!qK76!E0&~<7cPnER^Xo;eFPoyM7C=(T~D`1#?ve?hjYkHPQ7MY*FSG zOn2-PAG39nwPkm-&C~ zJ}q}p+V0B8ufl|IANyTx5ZQU19PzM^7t;9$T@|!PkvVFrMmb8e&S0tVcMNmQcfHgu z1jnk^6uZs9*7Tg9O?b4rn{Tuo&wpVc!iBnrr=26rZI!1)8_>7#l2*&PG8D}mLLg;# zFvq6Itn|oNE!e3b4W0@`V>O&7@XYWMiVO?Hf2azZ$o5x$qaTqilT+z|u0z4;t_E^^ z=z={fd5NQ>R-XMm|Fm;?{!Yi9@J4Nx zvz@uSyL4{dRxcDuAV7d51Zd~Jot^!*Gk14)_p_JxKgcJaJooeDxxd$S7lGZkoQtif z7L_$HmJ`Vb(H!-nU{Rov@rwQmr3uW!{}nr77ormXQe&dAE;=;y4quB;;M$v1)N!dh zkSUmnzg+zpxKuge`GQM@YNGN>~S(ex!>7)n%U zwX*`p#IyW#-PG`QTdnna-#KCen?rR)H-$&wPa<=WT+@Iom%2TM^2*=}6z1oVn*uY` zt#nQK3x+!cNGvx0e|+q^IoV)Ue8DF1jko^=4-%#3ZNv=y$xspBK9q;=CcB_b#8ks+ zKp(on*2af%pTLFW3v@(gwy{toqf>pG`JQ1QT!k3QPd7})YwA>y1VqFylJcM(Iho!# zvY#vFS7xo&tqmoFcZmM%=HW5eQSm&nG;%O96#Jm{6LV8e>%y_`X>{nbcMLKJ3yI0* zF2?cUb;d05ob#(e%ij^?!jOs@Tn*7ruFKf(`HgFJcB2caipE)1>(1N82UYkMLoMdE zVYzwPG~2V7wKjyf`1VkWB!Zn>S9g-Js@v!t&M8sEgnZI z{TpIrh0RqjawXW5eWm|Nc2oB;o;J&Qr=Drv8#MawaRamyJs#t!z#>EKa5uEJYdA;gb(kuY z2KLn52-YTtumU>@{aaUsEFcybk|J@c9g+9OkD04XS4qGUCYp0Q@Y-m)VV$9^uA$Fo zx{p?bt8vcg`sgY5d-fKCiQfa?kkX6};o0CIW25k-sFQ7Fm>Q}Tp-89rgeeKDyswP! zjLZ2p=m4~*>5dD>mBetWmC20ZCSEs%T#tX!pAx&0``M33H!(w;TBb2JhbDvfg3^#Q z80X0_VCV+XsQ*{|CO=Y9-*C$~hqxc6D<%+wukrwY2jK0odU1{T#WLcsitW*9Dl0Oc?1L?|zKkW_8U<^463D9ZJuIzP7~T>_ zcoeoDIa*;+{uN8#d=@%zUwsp>qhzvBB2P$+*Dj=pBDLr;k;b zXUIwV4*MTZa)byf(u@o6zJcv(zv-v(2OQ*7!o_fUj9S*95pf{mLsKGK2`}+Sb}y_8 z-#BWmNY)$+Z?4qq6C!yr-^4#&RO~{YGOWamW;i?0uiTn^qh98upQ3zXT8gW9TDKS;h_HMox-;yy+85`IHh@)e6{`jcsJG^pm1c@e*q z@t-!$KQoj<XX$D)DkAoD0H#76@M?x zCo$B{z4c`a*R?~fO3!?}9oZ+M2(>ikqH@8lpA4NAW^3=W)0lCA-TV^Hr9SF2a|;9W z3Q|Ie8Mi~Dv4ufnqz!hKsOtT}%`~(UnCJlHB$lkt2=|JP(O3Fc@`p2O_@@bt#m1UC zg0kWyml63>oJ%G$tJSB8f3e0D!?-1;srsj4HSsLRsG6fje3+>?d1c_DxIGZ#U{qI# z>4sy{tY)&e-o0_Yq{##*?hi#X80WXKYx98R{radv2upB z4!_l>Mt+8JgejGOg~waFVO^OG*r`A`J;_HJkCA5MzkH(Di|ZntQ@ViPL9XKuxPAy* zx$E#`_M#9+o-xhgW(xg?gYb>uCRorP6jm5!aXZ5+)KkO|dey7sP9fVO`{|D8Ufyd| z`tQYf7mubmkSDhG)m692{ATLvOEcaGs8~HeK)i_lFP33XqbGPjhmzQiw!ebikQD86 z^E>jHu0^<~5P^$At&M$x{n>ha4&8;L$(rsjs7*Xax{7-Q{tF%t<&%Rv!;#5KFPF{s z)>l>I<_gnU?<4$<&{G*Od4&debCt>15PuUY_P6)d_eeE&y+?H2pw)ho&mzz8Vch_% zzyGJAna>!#IQbeM^uC+viry`L<>W)cT{-?lbjys&E`x(Ix(2<&Q%HKW;snt z+?@4BIZywEc*gzBYlCxy<;K6m&xk64zTYPUs9Atb`fp-1KktspH*9gvDi-Ggsi{u zzG1aE%%CxWro&9S_A}lnR84nDlgzuxX@Gubw(&1{Fa1~X2Hx8@QRPLQs@ujKe3|2a z>>lhKRySOC(_3&B(jkZvqilsU= zF@k)y12FY2dy88I68m55_AW%mEDV^&kLc?dUidSh9#JTAOsUjQ#s@_Rtfheqr}-0+ zg`6@RhZea@bsWEvtL00=SKt%*WX7h=*HG&Bibh}~(-?m<)Ib)Y+hQ!oyx7`ORc_SE z{SC-E7%KMARYy^xnTLw}=ROvYw6}7V)vNOnLG^&9sB5V_)YibWP)?+1m8ZN60q_2Qm zXZjUHD_Rqr3&cwBfquI&Q`|!~jLswnbMO33%>%@=a6JAz^i&)#ndq$)Qwvi4hY+2( zU$m2RpvPh`&=23H>5OSm2hyCWscyoeS^J{~`akgnna9+P-MD=*78#Q`8M!v{LN^@? zaAS=kglHE>hl(Azt2izGT`oud2pNKD!T#D(^Hfa`>p-1krv#U3-U%#nBr9cYE4CGi z@O{YP>FZHie8t+4n!#PZdNGPuDdyn$*cb3K-v&QID)h4hE3=>TBcfxA)WTGJP-u6> z4Lr|uQGATg@HR(}5Jkd8oaS0VOVMM%c~7#oZzux4^fx9ui_?X3F`L5!yi24^IF4Tk zpAVlgZC9@iHlnhz626nsfs~O!bwl+V?-BGO_d4dS_&`3bC?z%pSLs{e*<=erXPS(8 z=y$<;{8uc(-NeU|Nq$?f((oUYgr3)D>K8CeRR-T%Vk@-Umka*Sn84IDT_TSQeaNF) zEMP;c8F(>G*vn_Sa}4v1|6ql|GA=hdK6{txCw@QJ1JU^2g!=`GgVO^=#s&K2NEy45 z6vd-HW$;$`lJ1rsC*MPy4b1$HA1llc%MC3eFI_2E1ni3XjmP!NV<{G!iVZ#CXW&}@ z8GHi^d3$TzD9CjcDv;&A7EDuNlkNjbvDcIfOetv#{gIH;e9oTEtaO|-yeA%*th^;K zD0QyTkViZ_!{ZgFg;>8oaEQPH%l+NeZ(VVQ#ofKig z7*Rm3U@OlCosI8H+(h&6b(l)n!}Ld!;di_>@Tc+>2b=yE+)9+|&_G*UO&%3Go9_6V z@mmAixycbDvI~|NTZbjcTW@#DsTEvoB!@f+{1@1Qw{{kUUov}`k$9%@tvV;v!+S}Y zV5|?HXB%)|^zX3a*fnN@sgp4UJgm)gbIol=&u<9m zLIE@iSK*Iv`@>p66tkge!bhYAdNcS|M_~WMJ95pvgZL*716D}BBb(v*{%YV0*=zM+ zB*O0xd?%wI9vl_W6E(R0NVhdqh9Wz z!5d2jS7qe7YgrsSj3Np{%@)Z9S?}?a`X(ZNBMq{_o7z>RpgbjS6!K>xT39Jpi z#KZ^NY?*J2(-r6+gGrgY1v2x0g%*myFBogG*{WtV5Wd2-M_E+D z-&{lxdt#@8JnaVKFAsXoHI8NDPKzd8k^( z&Bl}IPedZHmb>KFVAnFcAO=l$(@srlsMs*uKZiZ5USl9k&oqgF2SOkIfo+Jt3D7`W z%TSJO2}~9orcK&}id^J^^lRQ{w}+|+_W-8mRoFD8G<49>*)=$Mx|}Tj)0UpT2Hu)v zr7qI_=xXYwiX7MWOp!StF)}8#NvWVaG21M4nF@~x^<-Y_TPlq1pyH-?zHDbvA+VB> z%N~^-D?gM-QSVES$R{bU`FhZ!(-uiqz(uO@uKyZb$QwwHSEp>NTR4EJ4)}nmqlR-Y z9nFI*A5~JYFJO^xE}ky!2A)tk(%J*f>j&lMGwK4*N`CoLN?L=m;-$hj%;@Y`ogz@LXt#4UCC&d+ z{Z6%m%2DJZ`@pBrAM7w{gN919(dj8K>=z*o^-zAI{sd-k${I;n))JmW?*>cV54~^P zolI?7N_EG%b!$Uv8Q_Pr6^ruU!XEghU+LPVkU48O z;_bsMAKc%V(ZB=mQ))MO$6l+XmAWW%3iw(++cko?uh|aH$PdakOB>P^wtcYo99Pcj1A+buhKO%)O{Y$-JhUTk}O`17AW^7+V0TpbaCLI)Zd#vxysyg!8U+S-7MiO zw8tuEu!{H4CvcrbN)J~Yg4`KZe5Q4|@~^hZwj^hBcY?CB>0;6Br`8#lF#h@s<;4jgKBxoyp7Uo!Fye~nLdd`l?prtiWCVMf4BtMSqNvoQQeb{ zz$>IPZIDwhKdZ86HUs^HL#n^*^_|`9|0thRD>C*$v&uFq^ll0K$uUfwLS@@~Foj@C z)m7VEcTMHM(slAUSxcz5Ye~TzI6+q5(at@}+RQyR19$f=yKT?&9!}0y^^~`FSefC@ z&+;S6hORit8K#e8MCRZ2X0lnnlFU`kWschPL3vQo%`pmY?Mw%IIH!T*?Wh$|{pnJv zwt_C;aN#pqW7iBBu3Qg%vDUUV_YAOo(sTg!K)d0S@Kjq*aGVWNes56EYDnKuKF!x2 zo(*4f97%a24=X1qyE)3rp2$ z&N?+++tNlegxaOPA%6_8^kJ8sb{GAYJg5OD--fzn>~S@r9{^udOf;jJQ<4mxus)!g zs5&?YK`)^+dsKO*%m58`T~*h1XjPkCJTPCeP$oG3P%iL}vAr*Sk$6}(wLH&?WF7_U zY^tqlEUWK+EE`*TP%$2^3-q=0wx>%I-Dz;kisOJ=Faiw==LDx&!5Ls7q6V2`?#6&0a#_zx>)DDV z!<4gNAmg>nP~>(#lKH{)Pzy<);tUxCnLIn)(^4KfE4N4}-lGtDY@9d#uF z-B)Ewr$`?qexOfE$`n(q?<7~{8x&b(*MUaB&iDgBP=3fYg+1WvsUAc%ppDRaXHG(G z#Xb0@%_#$*%q{nAljJVTaY;jGDt!gYNExaaDYg*=BX7=KzDC>cEJEYe@38T>Aj9b_UC&Xu7hmq*zhDavJ)A+GM{^ZlUw( zt?7+iQmsSff|_Y+s3y?^Qrg?vxqodjIf|&o30}o3suTny9anX-q@@*xzx2)3hhJ(Hq%3k7Mn`+dR(Z`jkv@&wvoMd5?A3-1OlN_eI^hE!X>eCs#nhh`nK z%eGB2%2vng%JL~4Dg2nMbz4AhrhTUU3hnUr$;RbLOdrg^AwSZn`e`DDZ z02M!ME!CqW{psWOKE`GUF0;Asx(8&QceToI4LpK+mPMUsoU&N1^p30%;By>tZ7!S! zSi!A~C9jYBUA^X#0Jt3-4gR#ZPjOmmfd%Rh^lNyF{jBRxY8!l3maDj$e+%vB$(1j& z={K&W27!KHsI(A_x3%+Sx;i-*rQglWrh+O3Gp2ACSZ;0YoRxXfy%sn??_s`IR8ny% zUx0S5esBS>wR8dHVa69)pyz=JuEnxb;r%81J@K-Tt#ig~$_?03l7NGbQ`8*q6p2eU z!_}I@}EN$uR|- zT#iUn+1sR%<$xb&s9>2^Au;iuC);4v_sVo6kDu6 z;5Lex^6dZ)E&$eoouF>+O(nOP5sqW7aCs5@NBQ5j4^$QFBKK;|S-P)cHM23cFa4F- zqIhDl6FvYNt>vC^?m;H*Gs1_+ILZ3L9Oza zYFN2faU0OU``x`f^JMic6O~he?@(V&C+AFPKzxSna9%a=t810hDsNY^jsh}9hK9@Q zSP!~`@n7AKwL_`FX+o;my2w64{ai7xG|8T#>1!RSIBb2Q*+*}n@7Xp*hUZ^i*E`K4 z8>v`q6Tv^Jm6@C3^|VTUdh1x`SwZ`<-mVSW#=tUYp5upGBA-boRo82GE9Rx&fp17^ zD__axxVZ8(_zf5 zLT>p)=_9z6dPc!h)&EjA!FqcYx|(bv_?D^e*h??*XyDbrO)3Y7qZYXB^4)62mIt=U z80o#_-6lPZSTMirmI=gDSJJah@`#{OeCRahKQyo-I zt-r=~AVs}(r}CJWjCb2M6t`Dtqzys_)g5q>L;x$8hJea*(XtgjZa*ZSqdH0LYG9Lf z@$M>`p0R}Lo0tcTlI@CB&~0(oGn^8Ky%9aHL@hBenW`HygM6Zbx7VeUY~Z%drQ6`; zOnUB8=5ObNlxz4q)mg^O)J=T~P^HyVE5RrgmkE?jw*7V-vmbICQ<9*?ekN@MkV1Vg zZ&=-1dPYr(t;q%8da!{DQM_?3 zlXqPIzHoMV>^&$OfV3>?selq8iS~}n2{i(?$dxHuiUugIIm0#u#n5f+hO(2TdA2y@ zJa|hzCDlqFaLjOID;Ls`Ehx)!E0nXL1&aTaOUrtt^pMq2=oQ(EMU)qkF)9RRRo`o;)nlpV?gPpa zYP;o{^9oqJ?UW30o|Y>u18le5#{wTquB2a}HfkH@Z7hnH+kvojn4_J&jzfT+S*klu zSohkWlr&7KX&z*007sbhOoC#Ox~I!38|J*0tOr}VR;BEb;3*ltsmW4$ZOOQ@!Tx#l zzlrC4S1qabZSsHQtuo_)6#KEv(W+sR6Oyx32-0|lgBidb$45Khq3Kg9yE7SVgFFU4 zf`8juffX?-kIES0tO^_KbK$0R7uN!S4=j=m$XEf)Nj3VG$Q_RUu_Q)MbyoSm$_2Jx z;M2_bYK00pecM_@&Gv9y3AEgH%H3KLrE|2sTnXYW-+cQK%gfAGt~0JnK%u35+C^YO z@h^LS`zOyv&065C>%L2=lqw;pN!@sxAGpc9R2C@W>FJJER8u~z8tJ(STvjyF9t7*U zngWY~H>OS>Y_0>?IT8?ZrRW<#y z?4-*}*UuGjONmF<498*?Cneq^z|e z&I;>v*E*`ec1JpLThKMf1F3qrswCG_^@P{Zhv`2p5xdjg+Dmg8&%5Ffs*2-8~Ma}%5&foN3F6O?xUcZeruPg8rQG7c`5n`>};DViJd-O zbJEM4jbcyUDSf{C4KVGu=j(CO8VLtYVx#K-MuZnplR|?_Nk6ewvLX?w!5+j zbV_|Lwl5`EJC#gi9;g;IP%@^KGI65xNVR>`gv@2`V%3b|x_NM6UOti0k!lX8<;KnF zl0?}UDA{qA$yd~s?H1QS*X8>i4}cyq=U#1jSB=FnQd6*@w)0uNzo43)AYQ;c#y#f8_Nux-PrBH%JoSyVll7asn{*d+-Mzjn3vOk- zFB=Xg<$q6KPUmNyp!01i+n?^X;9?f(?%uQI7!S*Q> zEgR)lBwTkamb;X{=o`>C#YA8<7uRLSol9>7s@0X55;zFmnb{vK4c$KvQD`VXl{$SLghrl zYkEF4zo?D7w%R2RrwyfkDB_*B6*Ht)slyF!$yQSB%0(cY!Q02H+PG_Z+B&|{pIsj^ z?pbFlP-aEuDc2a+jPg&VwQc`8((P4&jn)^Es?e6q9W^IPW&lFTRC_r7i{(9iA?2BK zime1V1Oz0nRBEt`Y%I`@E`hIFYq(y>+5z1%3A)qPo)mApw0^U_E|ui^AZ=sY1f*p4 zr%?HSY2{Y6>{;0`W_mF!D+CVM(g7vd#`VTpx3G(2Ez?KU7Bm;Eatu>2_T#o?#SnGY zD>@NJddh0}CM}~@!Pin7dv*iYEd%myP`}gLIhuh^YQCect(@-aXyoWkWtAV!_+{(K zoTF@DYhap0boYZN+6gF`K3bc$pUP-lG{ssI>ane}@&%{^Rx6f)%Urcw+vSp!-6eZv zZRpJqKtshQ_h?m3aQyl)iZ!WYotd&tc7?QO!nllYRBL&r-C^6V+y$SMJS_X|NB}Bq z66GrQ`@&by+Ts<^TH7auSMGB+W**o^+q-}@C=0M298oreu91-fKByP5ulL-{tPPaQ zO|GX~t7&egm~Dx2j}x}5(nibwrEjJ`whyJ#Wus-y@=wwq%9_|em)}k505o&IQmvD8 zqAw+0D2EZ?T!X7s7!FT zdTvQ|c&VhX>w){CW0|v#?bo)xP<`NV$@AjzN&mA=q<)kY#W-Kqwmai6ZFAvJu56aP z82ntkLOQ1G2KCIjl{U-fB+i!&&3GI8RyL-MgU8CZ%DdV|6gGl?+poe^mE)B&9BqI$ z@{gGtEs3rV@+N6bffmfA%rmZA)C^|{2+BK@&+`1wH6_1?ZEg7^>T_8O`nIyz6Si-l zcK{cl8tayUgk(4%77eAx7dL@@wi?vq;+)vNssy@Ie^IlG)`L}7jCOB_y4mW%|4A^$ zs8q{#$(quryE9~W-(j6|+b%FW_^paQD*1Og_cG-s4-Bd0q*sJ;` zHE)%HFtr`30qq3RnC^-5$`2`&Wt*5C@-@oAnM>rWJWtBi_5_*I*~pP@D^0spC7h{J zf{K;O-Ljg+y9xpMIot8rztEN>s(cvms_Y95D~rmemsKyFMjf}$`gIEvjDL*oFg6k?aHPrO0Oek51w4)&W zc({6?qcBg?ja;g;?Wk9I$+S$&;n#+Y926eMFV1?bs~!QAFHCX7-AWI3SqI>o^_sv1 zd_kzOGb!65J~v*#H+idi=jvwb-$#csYTK}i6Q;SyPI8pc-n1TXt-~6$#Y0UdbMLq@ zqA%uA>nKiSp9o#>pL{MefvqQGMQ&!TAk*v<@fP^S?4z1|{sfnvRh{{$&k1~`7aAkx zGuc=1zf@gF_YDUSqq)cEw9psh6n;5&!Iejx2p$amVI;VA zdL!A$^q7-{ev%E*b7VW;p1K{$K(q=M)?Le*qaTlq|ko3TgP@*h~y-47ChIuA~4cu^LN8j zIFB$+dtH1LnrTfioDMzCx~#`4)@Q8_#;m$1C5{y>6;U9b_=+f_me3>jRepg%h(0rY zhMn}6U`Ue+ehQ;#ww=}dL#C@EyeeFu%`>J7k3HwfafXHQ4ShY4V`7~7Io?v|({#np zfwJ&gaZ=<3ANIlcZR2y_8BK+Cpb*^N*;weGs(mhg=T~EE!{Z`v@v%z3%TFY`D~OU% znQ0;aR%{hM5VKQN@$}|aSM>2@5kcg#x4rQu_()ZzC^8Pm8;KjiU*c?oH>-y?2Yu>& z9f}J3j2nZ!rM-irD(&h9c)uM!%>!~}@%ivK;|>2%Jsr?q>Z=)>1y0+93OqgHP=Q0@5N8v(qJBvZ#bZ-DZuy(aDBx; z+WqjL&`GRnWG0l__?^Bs{)MZ?&8rxWya?727OPa;C$yLwq5B$|ZukftN6s4lD6OG8 zow^O1Ntnpj>>|;WI7;`@*c&5t`_ki0v0gd)B9_AhjCyh?za9->qeLZk3%F-YbCqTl zlV6bEnkm5oIvtj%5%VYWb?GI2ystKU-t0xrczZ@#h(kj|2%UD7D#E)}^YAQmmmn24 zN9XMZja%{M2GQTB;DW21}kTzzU;$0)Jz!GtI#ZzPl7Dl!ZQ==`I zk1^-hCZmVk$JGcoG(L}%>fZ4au`GQDbWn7;#%1hRv4F3_e^ep(aqUVmiu|EzCQ3YW ziAQ8^pcOkcOltlYnG<6{YI)j_siwX}6GaRD4B1aujoxx9$bneT7!Z)HKOadm&JO+? zq{QPy%~+3FX*?ImWA_nn#qk1en8h#gB%?;Ywzxm=gKJH`+rAqai~kU2nfEb0!duv` z@HnDrxYAUUiWl!^dp(D$4sPyBR$J|YGM_od}5>|B-_588@j!Pv1<51c!h8$;@4@@ z3yp`c&9Ow|ZM3tt*i?hB85mj-HMDcVL_6#*@mxF4Jrr9lYrd_%fe$|i1_+ZixS^JM zx&H~8gFQ8UN7IZWeY?U2_(*s@#~3HdTBv={O!X>az0QvG)Sif#vW|ve?M~fWc!_=_ zni+XVmWd04vpf$x`TUje6=4QZHDJzeAzN1Y$@tPgPGb$9;#wMUG0A_DT}lw9wq`T3 zF|>;89vp~oq84Gl@Qtby;Saq z$>-b(;kI}WU*TJ5t-ez)q~OOQwX~PRcK#VT8gIe;$m&eiv#lkA<}}y-ND>(j4JKoE z9egyp2U}}+j$e=LApD{|vefXqJQ0Dpb>i0Wh7cQMvOXB}9g==Cv(`~FV)kiqNwZi~%3Rx9;Xg8y~E4nuRDcKjCrXlei zflhe6fLXOMJf7&KllxW#hX_h>dRR~B$ZFz9A}pRI597aeZ*(J#+sM0tcVXD{hI^}c zM290*ZwLQlK@QMFLhyPlS>zW(!nHn~bEKm2Eq*T&M@j*8xUc_K zWVOf;qluq(BVw(vdnW{sL|*%oBh9iphF;qzhdS$5M$bX-!Yi2f`rp3w?k3E-tgEIG zTqZfv(11_Mx@&g&+X^GO1b-UdA9)rQ^+b3SoNL%mw~v`cP3GwpPxXE{gWrVh&c25S z&`{RFAYlX|mhf^9A$E!M4O{%JiF=`W;pSzl{C9{$;ZpuAMv<_^ggC?Bhz;a4;}XNt z=%m=1x|JGr_pJb2!_7Lq&X-)?H*vTB6j_H4Xb*-??^JWpl zx|sG>cEbc@YJICjhV`gCLDV5;XMJm=@W1lS%-Zi;f)zq8{as-$f7{d|JUejF-&^>L z9UyKX^2~nhh`K*|AXayuV=3rOe|h#;=xS_tJTD#wzlMhqo5G`XCUyaNP;oePom{Qo zPDc6Jp;yWsjzh0U^wC>X0?hQABOv2=3O|6W#Vim;xa75lO)CHTXJrl423i5xrF3R!EB)QRJ zqy=j(E)q!?G?!>Qb0eeK!MU;OF*CGFFXgudensyQ-MPkkmH0LG<$uCU*=7+jc5<7* z6mpYo0p~q_UeHu_Ham_x!oM?mV@Bm+feS{zFkdI zRPkAday0&vtr3-lC+p7}{|yja6>=lm%wMW3!bEYMe|@;SzAwMU-xBcoF6;lm*>D(W zfGWj1A;xqm^bGI9qx|T|SFI;t3m;N%z^eflBNK@ALXqbKpXq%-{1eJU5Hd~N&&{UN zoudqQ#p-Nj-seDJFgt5}6!msA4kNAXNHP>iXMVU}5v`)*jEQ77<0$fs*@PX6ELFwD zjBV}1V?(R`shZQkB@T_$Bn2@NZX?MGCsxig)*xO8^Z9S8-4!qW+6aNj^S%qw#%Gve zL_34he>wJKZ^`F;7u8+vePEva6)iV^ET1GiM-qfKMx!o1RHok*B6vaP$H2%C-P7RA z&`rKR;tskhH)y8_Q@N-qK9+Qd4-YT~xiz74dOTbO`V$`+BO<SM(1TG7jdFCQjX&WC*XY++^YM@ap^p{Y&ctk_as&I-?T32rz6@ zp)|BSOA;C_W}2pg;h4wwZACX%$n;f}V*tVtp@X4BKuwHorB(sV(yfa*t0%^CX>&rK z!ip?uw5jVfHiG>bzD)G-9yZfWd}5+L$FRqFNZW%f);~f-qyX9E-)E>vyokd3%`tPD zmfhifz735I13Q^F`Q~`G>#FLj`!A9$Ds~d%$s43fbckcsJ4lE4Ong9JHf_Q-hi~{H za(Z|W=`sEfTN+DcjgD%v{xYVEUhy&f*_lp0weyYth}m)58^&c#_Rkj;yjR$%80haS zj`j`K<}xpItML!I>`((P(eyHMuEH$;9J*ckw`oprwf3;Fc@`v|C%c+% zF)v~>3i^kRs%)9zBW56Qbc{nGw!v@5yg)zriWm*u3u|Ovgw_bxkBsx(CNp`S9V8q{ zb%{eOrVD%gRfQ|;D8o8DNmm``ImEXCdk_flClGsZk1&?*ZxakJ@j*f*ciObB;;^5= z*EC+P{3FIBg@q1OZ?U6fl(~&xpY^A?t1sxE85|-`HAEv)PUg9UEaOwN9_b6+Jp(s= z&mxWWLkt%7KCupr;xGI{Fp9=S=7pMs`*VG@vqKf)YZNkGGv(3sv0QT%l&NyoOL zn5Urz3pEB`D)i9g@HL_$cu@b`&?d$RRgdM`akeJ$jc;K#njhm&f)gwBft5%>+5fUy zqXBjc-oaci^xBfh4aC|+FJZqu+N_{{Md%l*h>Wisl2tEb0)IaA)8E|iGsZukK^Ez% zAiGTI{(Pg=Q!~`7qOGaA=AyAm#0T{g^J0`na%hTBU8m*U;u8NAu}UUxaB)?21#A!U zqj!?CAbQs4z=i_XM3JLH5_4JUOfr_V@{b6q(BEVPU8?z@@ewcOO8>!#l2Au1n9h|O zf+J^yM7#rL;!a^X%m(oYHMjDPSBt$i%7qFJ^M>p_#fiac*kYowG27f%^9Py;$7!1| z8O952Q}LD}?B5)lw6RN=#@>nN&{oE#uEjaX0{8uLfLC? zd$a-5S(oan^qaiR1u6H-@RIA}KgYe{NBL^=nOvHtzq${16Im6yO8|VPa6gtP&*uN8 zUlT9Hr=b&QXJR5Lh4tj@@`q@fn17=U|6iypFhym<)``u7jIs&R5sUa+^CS5nvBG#h zbTHflg>_c0w^3uNtuq@q_Mx|WxE{8c%a8fQYkK;bUTf!s>jc+^AM4J1jEZ7#$vl- zOQyDAS0s)wi^t$i;Uf4nY7Rk6Jn;^%>QT8i`PX2x$j0bkzO4bk7Kg?Pia;&&7WzEc zgAlPI{t5b?8YBG8?Qu^~)bpLgS+#}xh}~nw*ksBohJollaT2vVyaQhum|~ikzClw& z-u5WS#fonnXRYMMS|{+B0rM>}irg3Sq5e%ESv&>b;6-#fcG;^azkok6?#14rLrDcU zwLuMhO<3ycU8Om1;cS6Y-A=t%KSB3ge2Om(4AA8Wwa7GVt6GoDv``t^OsRpXEpTWU7iSj-2 z#D6iRN{*Z}^hdwb-&Q3f8_8ucpN-i!nZG1dM_1!F^)XwFa2p@vOZ7it8K_uKcB1J2>qj-j{6QkC7;TRLg zmKR@SDe+%J1I>5tA{--D!%?4^SrzWcPex;QWaj8lmRMTz>B`jQyUb0a_1 zI7~hPlg4H|u2&Ai?&?nYA~9n;st>4|VP@H+UOXy%S05o~hZ_af zh1xN1wCw}+z0dJS*xBG~wpK2RS3nU1R&pq;w~ z-N)0}_l~jn256r8)4(3=c<*wyC-OjE%X^fvF#)?n+g;rOX-OEhuYJuZH#@)lgSHrc z?EYeZ-C(Zr2fbH|GXj&N+2~wo0lbqmoMyNBxyM-c19}4u0~SF&*mKN7$HuhD)Fhd- zSOQEoCz*SI&$l;oJ=J~w2Gx%(Wvn;y-q^nW5ko_M&QeS_>M(?h9;)XEdoAHFwC+ifXa zE;U#;T&-)+QhiBN2kwp2?hQ;;_+RFmCrEEDUx4a>8S;?4ru&YgF4DxAtO>Y2s$EQ) z-%zv&EAf!*4t$bLs+TA$_4(eRnhZM+ZKWNm*W12pvK6bXt-MDwXQMMzf5;j;y1@^b z9A=Oo(I!Gfhrk^4&qN0MwDxl3W|>LRQTGq<#?=ToWuH%GYnM^o=u+=A__n5vY>D=f_ZEdi z=Q1v&bWdyJ9ZnU&x1akM zinQ^8G226ME9#u3J^NN2QD)PV0&UrWj`QjS8iIZ^JKY$yjx2&^>ccEeg&Z|3G8Xat zELegrL=JoFyA{O-x@PQ64B~5fZ^~W%hVX3LI4$BzQ!D{)I#-l^%xt9i#Ga{lPoBd4 z;XJcVUi=T-THj2U$Bv7vP=AN&*!p|+d+I>dq1CGX7EV3cQ=*Lrwjxrds%)t*13FbM z!rlD$+1%P2$|k1IqCPVtpjz^Kx%H{qlqrMqRtFu_>Eh||3)>@LveT_PgFN&M^?ZZ2 zIGmolrMg`qrLxVYJh<4 z*tYAAo$#sBb8siy61pYT2C0EOSH5?Z(VvwcRj0s*-d4)^vd%jn+43P-Y%=N=zXEKG zw3GBj{&Y;V({v|~g&LpqNwZZB`FfR)wLeoo_Uw&LDOiToqLqpxC2N=ip1Gc@c||do zPjk4vFHX_bQ|c&Ux_FYs!P!QFG`H?oO_$ zF+Z;f zKLqc4zeq@1@>VG`oa)CUE2aTNXsXJ(JzJGXZ33Dp^=xp>8#r5g2)!h|Ut9*|I=|aa zGl$`hg@g@PH}R{aGaN|9YL`s24e8D1!z0T#3x(Uxq+VunJU&LLTjF}-tf}e=%+dXX z{}dtTRPC;~M6cfU1=&^j$~p*mm0w+PU)02^2+MM)fs$_MV zb%J6oL|Shot;l!;M3kj*7nM3D#}O)Dks0f>0cFe=g-(}giMJoJ{Gv1IHq4k6TNq>8yO`y*1ko(GUMs$e^e=6ouBix1m+~HON^suc*B^t zV5nxLW_4^XRtfdX(=|Co(nXtYN9AS8+Sc?~eHzKgu)XdkqRVkYv(z!vxl^5venjY^ zhJu6XugO}`0i;qsqzM@VnWsQ|-D3}z>VuZ6dwMpbCA#N~Suv6MkGvI|U2Y}oF~gXn z;16$#X9Sg|{Rkt<%a*C`>%habyQ=fCZ|0)xg}RXWPrDdsq5MUTW%<%@D!^8GPCau&5E8{B+vp~U8Eb-F;EZL5ag8w zmaXhG_cGhBqRWv^>c6z@QZ07q*|3j z6{9@ekd?MVW}~}yBUM)1%JbeoeIeihc**fJW3_IF>pT+xd$?BGE(6hC%+q|d7ks0w0p0@_#;yaR3hRDHEe>FQuDQ0?)%hdG1Lql__@=fomOk>;tQ zuPohDr1h&WFawcu-aXv5z_yIh)Wou8>_)Vc0)szD{xD2cf6F-REY|#D8q@8fvpkz+ zx5|$KcQs8Zi8GuO@88ShQLPXoQxoPZta8?$1q09r#c=P8lEacbjiugNx`|f+FKClw zW8p8J3GAWtR{)Oet=Bqp6|-A2z&kMJ@javdfiz&|Wi~8tjQ-X%*_sRt!It~Zc&90x zDwF57dnkQMS3ha7c7`|T13il+!+>mfxW8O|Byk9H2)-iUrK0TfWr?=;U~Ez+bGhiK zroW_F%*DJ&eT4oCsi!`wC}O?pnQUVG2XC%tg6e!Gq)B(LR_s62rSwYrEE4!>rF@IFeeI_m7sY(N*e_5pR+3U80pTmBVRmsSG(t9hYW3xCqM zWH+b@)JPQw*Mt86a|0yO5`o!r_HXx3hAHzRf7)7=T(KjJ+1moxLBCPEd@a4R)U|`B ziq@4DNoybzeG}Qk>HkRM{P#5*;s--AV#0M^V z75QQ5ZJ1Tu%NPJZh|SMddNs-g0PIB#(>(FcdSwipDY<5e#QsvO+Sarh>BFuDZz!|i zv64oPX%-jrueNT0{njsor!-;=NiWwEfFHyWRh9VnR(ATX9yx8R8(4# zsD@I@<=Pr82@)x-H55Tmvdbj1oSCz|XPHCSRN}{0}9qf_!jM&V}@2u)*IR^bLiE| zm$ag-=8p4LtwmX2_`<(}rSN?IO`#0=Ku|!Y^-prYDHdBPd&15H6dv~qb_5ye{gPlc zkWUDo_>eqeOoJJQKloJoy!cdTRZ>j)k`=z!h;R9@Fq`rWZG&kwwe%K8w>FU;4*PL0 z#CMp__r-Jn@O8FCE<`?t)$=d-VF68VYk*DFQ^^In#weJ=%of|lMEop~MI9BF3-=rw z4X3d`{cE7#wg9E7Sm!%nH(;>k;wHKUV6G+jc2h>}JULft5rZnX?U9KcLa_RrvdRCI zZd$fT6NKuL%lIg14SO?|@Sk!uVV$-%_6hk>)MhX6!YyCguGb{lW z0sKmPKt)=t)OkKzo^9xhc9YA*bwWI-7Vol!!dzw}afDgsn+IEbh4^6QRNYAVQ)MjI z2Pwhqpjcc;UNSqyMeusycfE?XLXTyp;Vt54_Ege(pMhB{q@sSYgHpA}f){z6noph< zXgo4brzI$Ng_nvW=3lC5&sL4GrfadF!P{Yp#5ODcp{nto#(_cuJ`7xnnHRfVIBZL? zoZvAvyyl^E1_R`5BoST}>xdpOluFWGV?Kozeg_>)hZ9|vrJiN@lRdQOsE}d^(0i3m z{slPazsU?_YfL}Z4378<cUsWCs z;sWf)5G7#8z|%lFGYKMwg%Tw%2RV4R|0+H}Dg_x#5}zcyY;$Ft&%@%dOrjIKU*BmR zrA(v#F6zb1#O%INa3nT}KSQ;mt>!RcA0t$~Z~t1X7jE#2)gtw!F`tZLiojDR5pc79 zg5BBWP`{%K@Y_-XtXJ##g~D2DFP|g5qmh|pM{i-0K`dWMHDH&^cSb*u z>+vCv_tod%FT?PYLPbzvBU-ME%ger&$b+CBVw^i zWsQ5Arw!Xo<%w6kEA~$1?R>4F1V+Kx-Yn1|UsW1CU*X*{Ql+E%s}scEosqV8xdHG) zh7^V#Y^1kR96pcGS!ba$P07SW_z#`$8Rq7zI)F(!>sw82v42bU;8rQqxq3sne@;ZD z%rRN^cd7H-5haM6K*r7fNpZK8K6VzvhtuUFaWMjP7nHe4{@d`w92Oal>z=;Cxw*E!8 z7oSItdg5I9U_TKIEQE&kGIa2?H&pC4{Yr!g1~`k&wLJoZt@*Z(eIdjPYd%wsI+gFy z&d^Bv6mmFr+GUbLWKq*m734r>8F2x*gbr8B%6_pox;^x4Ee*Rmq%T#){32!>S2^w= zjNuJJ=bdA7+8T*&F+a9ZdF+JkSB^Tn3HyN$ukU@e~R{>f5D`D&><&R+@gvs|*DzC)eyZn&I^3q0 zM^G!_aASqg#CqftV1)a7X$suosPOcoM$-@Z?bKS>Wq9s8rdx)_b5|{)L?}(#x;+W_ zSgOGM!Ztt&Cl{chLLzz4TDa$_d6(KLq*W(?Zl))eY#mb7Dt=B+HfEwx#Y>8-rRhSd zd;lLxmQoMUoxVAd8L(3ugzVIkgoosI{s1?<4tCOka5&-d7E3Lzhy4DKgE8Y|bDpP&i=^sszl?!+kt&KVlR&qUi~Brtuqd`K@t0|d zkykc=X72#e?&O62*y6zCG*WDk-*&CH_GjlRa_9(KEE$Qna3jo#Hrwx`y71W dzQ(;zxZ!(p3Ub}jC0{7&Byjq$_rBc6{eOlD|9t=e literal 0 HcmV?d00001 diff --git a/wasted/disco_test/3.npy b/wasted/disco_test/3.npy new file mode 100644 index 0000000000000000000000000000000000000000..7d00dfb6116356e50e2bf758ba2b63438fd32ed5 GIT binary patch literal 35968 zcmbSSWqTC&*Tyv=n`E#{9eq=HRn2V%gO8BuUpSS<;%@2x5zg3*I8p`+L|D?rsL|`T!?L4`pnrg zM@<`%K6C7^|2uC#YRar%%g$#_7&ZOZvU{qI3;D%`)NRnPw%vs+LH-|Cjen@6;opdQ z*ltG#GSQJ7)$$GDA)W#%fk&}_ivlU@C^vDj+$yb{_=~fGHJK0le4$f;jG#~M7U*a8 zMteG1U=>`IORfrwg3BUTv}REeMeWbD4``jxEHc@5N_}R%O4*tGV0&AZW^in>^-$`C zmy*v#GIGg20BL~slm@5~;F_9C*@JO#%?K(=4v2recAev{tx_P{B}Sey^PJ_W8;-fI z_VjpkAa#QJJM=hUa>d$U_oKjcwGw$x9E9vv2f^8{x%kE4sVHy#SN((N5WH%etw^<( zgxaDcT0iwNHWomsq0U+PfoMhN*Q6b~{R#Kou}U4~DO0V^T-$N$3;93F?Sg`xypN-P z#~jy8tx}i@)~DyO&x4iJ3ifyz(FARqt9c-CtKPNN)|dPweFXc62=d(5np`KmbHu1c zfkg^K&es0rPy7BP-iE>iAXjk}aFx|XQKN7hU4fK3x3UN9CPfA10POA)8llAt0y`IT z=D|W|Z7B63qN(+KlWP}Hc+up@2Wc{K!?{Ie+)JVn_ujx@Bmn+^`O|6G{`0Nl8B+`Pa)(_HWXkncc?q4A1CE>8^hl8Giv1ByNU$3D^Vn|21N;Myk@yFOe0Mz0swbPaX)#eLTL;V1F|N(?PvPiqEcl8i)O zsFmbat{lROT!_0LwnHr)SQ)-60IrdoF5KXkS-~+NRTsw6v zHJUu>;>k(cOq5l5LE-LDA6%$oh>^3GvmXVo2I;^lqBX(7cDYy38Jg^xY5&LkAbgs9 z!&_ZnT~lq%L&?~`wq4P-?zD(S{)iRl7ok6*O>9P@ml=&yB9ImfUyjtr_iAHOT0k`- zI_-gKBPJpXQaP!$QZY1u&nS!9Asei|aXY=~WqC7Q+b?c)=ebYg(|jx3u_48=R2fek zwKtKxuHI^MA#kW@$}=GvK-!<@hDF1Zj?pfJIN8 zU?`e~*0Rs3vNNO?E(>n`MR+VL;$3Qw7kh^rv#+oY%IK1d!4$Eg9Oq0_mf10~w-RS7 zA?mrZ-R%Q*b3{FXtRhxwB@r5=9L7{F+$`8pJqXNJ$I-Q2Nzg~-6?N8m(^exipYJIA z3<$OwniOc8|J3`@os3LZH8n1B!qGy@p}r8asRgd_&>2@8+*&Tdkd%D&oSNv!5eIx9H}!_%k$ zxl?Tx0i!_4S8QD<(>@3d2|sM_nckr~p-f*}WDvJXtsVIY&jJ{6a^!OP9^{|=JLkvZ z4KC0ri=)*8!Ixo8qeG9dqgaHmW`?!e!Ae>q@<*!9w1W2$UAR%sBEG6`nAY8Xuq@}E zXd?|(iz*b)=dJcBZUg>Nx*x`o*XEwU3+--bZgFY&*AS+@F6|Nd1me~)dD+DEL?6K` zC)8eKWeT>g5UQ)wsGiaiqE8^v-oSd3ek5Z`T;#eVnd@FSP|9+i)DDD(lUF0w{8tfu zk;#$A?LioOCN;-8C~%j$Zc}-kHO&Fpx|B9?)lnYcPU>*b9ULF{PCX&d7w@odwgJ{x z#J_yQ+B;#A+Rhx(Xlsnn$I%YUBL1-7Q~M|ZbRGFYK9BAvR-|(1R_q92Qme~^kS+d4 z3`{wo7(~Bic^9Q0>`wF{iFS-)(=>(&K3Vu>+rSY z0q6!|CjQou?u_$Atc-J7xIQ`%4~Lt^c&O>ZTmFf!`bua}%2M}B z`%HJ$S}Uo}fqm$E%HtoGYIVNJ2ORmq##B0{2G@knIfl8*hu#J^axckt!~=VE`+8?N z>qM%CdM%KZSY&(YUMbAC?vpm5pUC&h*~n~o9kn5}$2BIJ=umR+h630(-7}$}WKqex z602|wpAotX6$L(tQ!t9TOFoM{mzLYyMEW7)!0=h4V(qKWHX z#2jiVej@sxxv$YDiO=?5f;Hf=)Mh9N3|co5 z735CbpztPjpY)TM#GR6tMIKo@tN*zE6B@~pKyUkI2@Q^h8&RFLk2Xu;#n3vjnFbNv z;c)OkDAo46UFk3odSXc+mO7`rbnJ>|hu&g$Qri?ylY)-Hjs$Id zaI|s?j&ZIjZK7>+w!+`IC&U1%NB${ZsU7F+2+a@=N`0yRilcZXDG@!9e~@dgnaWJk z;EaXxgEiG}S}$fJ`U%kyzspynFGHz;Y2;V+y8Q~a4X>*%qTUH}wR6_9@{E+Nq4xG# z;C8AYWER121?eocPEDj#(4aOZ|HB4pQ7d6@V%rxESS#3`Mr|ot@ebA#fXt_Uo%RPcl|6=zf z!euXqR7|R+zN`$ZtP4U8GGo z9hpY03aoO~AvAcorQsL;kZI#ny%de6>~? zkD5Bw{O%k_WoyppJ*Q6^V}IzFEMx)GgBj5ZRMlv26`?OiVxzZ$Mzs}!Nsoev%}4LE z_LAqJtK|o-R{U$Oh8iM<`@Cd|>$TO5JaQew*N9o*7%JI$6u7~UQns3P%6n%+R~u_P z@TFo4KP6I9KKPRA7Eu$_H^CIyg^o}UIlR{0ls>W6h zeS-OwROW{>-Px-M2#$|TR;Sy0ssKIKwjdN%cRDJ=IZ&g}Y^@iy*ftFvXRF{ifDaB< zptq_+iP_{pzJH!R#At^jr0u?f6ApW4d8DnK)FQMcXb3>|1Of>iLehv5tvfsb=t4D! zxlcZFD~?2yhF5;vy!~i<)eO8+uBj+D1I!I=(`>LQ+wb#9`*<) zJ(b*KNlUt1cmj4ooa~tFS{poO`rSGSyQjtpH#91^M;#F^5g4kM^+~{$vW7TI+_Sfo z+XpHtHv0msyX133ZI(zC!WCkbKv8*eZ0KKGXO#%=Bd=k9xvQvA@|Y|JOV|olqrC?H z(ly`x4^vm(1*J)IB2Xw~?~iuY2^T|1>+)>Tz|Sd5wDT;)Jbbs>WKyfeg^)LJJD}~_1u%OrLJK>l}L)awtPtFoyr1H z+Z#JzTcf=Tc5)W*V>~~=9--^%=HfI%JN2{eW~2%AC44YiJGfUlLk?j+27khoP^I{{ za4WWDq$$yg>|(2lPbT$fJGEA@AF|H5(3)bur}c5+`gL@X!X66rvYI14l_!KwD7FVxoYLD^=#FZ}LyQl*g**89p9pv2(~azR8h zW9^0O;&No9d7}D0M34#gA=+!KfqG9F;&~u_VKCylxgqII+%eR8+eVTrQaH6T&Zqx!xn_e9t^r zH@P0nYDTKJTu~YutmQ6ESw}cM7x{+j7j!b^L>D`qa-&G6ynd1IuASO^XfD{Dd0^TQ zAng4Djcpm$fl}=Vg>xQ^-YdQ4f44pgeF&V7ZvzyBje-6^Lor^P zobXLeuiiIhPXt1*AWs9CL`{W@CWV(0E2PR~ZTog)tE(Vb&)HdgX7A$YDBaZ0knhTT z@Q`{i=nb7!2FmwsMfOl|g10~R2;QPSgjbQrayjjeFGFde9_HJiiGfSnIr}8CCh-F~ z$iH>!pLM&8)81(H~hPCwz!vS z!QO;V$ho1j38U1c&=i!ELPBG1Hu)3TosemZVc$_5or-Oi^=#lkcoY6e^^rlLzm&tj zw9SP{NFd*{MZ`3@5;w&lkc57{)L8vMo>4DHeh}rIqm(AvX{;C~hz52dWg(sm7rJkd zZ)~fP)53V{sLg8&k^MuD9PddVvk1AYta4mUJ`1L(71TxHBL2Kx@5UmBDHnYvn89bb z2U$N_c2hO**G7FZ66v8b`8|V+Lk!20Uim-!=s+Fbi~Yb|sL}3myoY~KYn|nR9sr=o zq)y((BnG$3effHY^<2L?hUA_y3=aatLg9U=wcHH7YkRKtl|MVq+5c0kQlPU_s1EhY zQ7e?G43LM|zxp=>3F?8CP5ln*@B;0%RLyxhGB#8y?N$?P*XXVER_ZENsOD2n`ISwu zo05-&8b#jP5BiU*W2Ix(CiXf+4`Mwr6#Ee{Sg)w5c7%ce;{D5qVuAB4{lU#T|8Phu^$By`xOQ@dkd`8Ez!K5o*v%iARTtHhH; zD=aV6Kq^PJ0Sr_Fc!&MpKwaA$xwm%IWH6-ipQNtjX%r6DE9?2KD0k{Q%0Uc^#%kxJ z)G4r!zgB1=HP@wcwc>Uc?@>m(+G<&p zBya+#c+wO%>)uey1K=-^dF>9f>x?dJaH-x9)XZ5|Qxe zPzATbyeBK$;-jX}{9s4wDwAkTiqv#iY@KMAU1u$Ej8aI6Y9ooW z_q2iLTIzXuuBX40Zd;SGN&eq|+=q_>AE?`?FVNgT?XW==)f+0UO^2683`9kWf;c=) zJ*+%so9B!P8kl(dXC)gLmHbi(_-MO9er>E0x*4{ig~6?n!>o zvfpY%uMjcl-)JkPk#w1EhGt6l36h#^dxC8>u}Ihk(*!UPx&TCY+_sgJnJz>^QGe5K zz6{ia{7U?d|6#9*tx?kDVrG%;2NE`J;x+_YSrfTN0T6EUEG-%uQxLG2Fq5+xgbSXQeg~l5#Uqq7FKN z{ArC>{9Y@#iM}4LtJF2-VMWqJ*eq(eP9AQJ#1FLmQ*fv7D0_~~LM_--WE{7#vu~UTS#M$etgv}5*NvGW)gTI>8{&8kBn;FAygISk@tQ>?gZRWMC^qDj z9OsEL#h&{m-;7-e8}-W49=4EU^2arJOwJ0fidG|HxLd*~Ap@KQq*y-YD1cZ!zwD_K zk%68!`e&wL=qg=*g2cXgj?+7l^~_o@Zg(5wV7Y}Shz?>N!@a3kek;2fb1=!sX1PJ2 zA>MJ*X(TrAF96$SVE>re`neE%LeQ0yIEfV94rQXZGC4UmcID-X#`Gz}MT;fwPqz8JJTGuVu4|jgE+Bh3e3FDd#!2E5F+jK67`8QWqOyXI zW`_f(q)_H4ydAxmDB^JCeP9aLGa%?U2Rz6L*De>g@-KXsEz^Jt6F?fBLyS;bDx;Ne z_*kwNd0aND!--}5R`533L$1RfvP}<%%@Fw#t4rz#cM9(bzAUWD|fNxK@ZC3s{*jHYMPq7c@ zfAB`~5z)z5lHZs-kM;r{7%l@xsE<%>WFO90w!<}ov(OCU2AytBAj@gb;UmH^Ya@9K zb8O=d>pM$zYz?`G8;c9VdMs+qmA?R#e2IN0x4?{2)3~F)3E*Y0j^Bec6!swF$pTvx zIM4YYnH`N*OB!jYgkB(~lSb@H>;z;twhOOeg~-8JOj*60BW>OLk$7FRnC~6hS(XoR z%4c*OyPfE1IgO{8B%-l(C$L!7=eG2Ypi9X|($Sp(b((sE%S*UJ`;@7^k4RAU(w+Pz zYEIHD{vF+_V3st?x`-Tw-VtVt3&fLw(@BfjP+7%Z15Q>@rmL$HP6+n!X(|uD%)GTO z;|b3ozAf3^`oXpcXc?>}RtFg6CN_d!h4mntl<{MuVJlXwxIv@(8K#A+!PQBXZKvR> zNCV=9k|m86s<3HDMe&=xIp1&375rQo&KVnRY3dp%76%Ydw0V{c+g4%}v6ei6yyMR% zmGUIATArimD7%ts98ZVAcJdf1nj~w@|xA_BQ)Dp9kjdNU->mn1dcET-~F5?gj zXpwD_{EWU@bdo&lI+o%wb+831lN2P&McptU)E=ISn~g(?BGL<`mC>JMq{jG5$%}kz zD=)dS>j9(rm#Bey>er*~`7-2JmJml4m!tgeBoBf~(lp7Zu z#hP&$)^2`6jNl99^@6AHiu|MC5uzfPEe(hjgQk9RoMncwn-CxmaeerCIK==$1LaDf zlz)%(W*YG|$o|+FuDa3$oIvJ_y~w@EcA+}@72!yOjSxj7S8B;T7H_a)@naZ{8^tbS zP4eW%XTnhQu6&B?XnUg_A?zU=-bnaeX7NsPk&yxygc{m+8Ty$n#>|nw^5;z5)h>Da zBwQ^G1dtBwaw&`q;Y*dF(!bb zuiPU6A7i>pGYOSUE4il6GTZWj_!1%$eA@VQac_Av+1kH9 zctg0WOoSdQX1b=jlsF={6gR;vxq`3E+Ki2Jowjc5A-=24QqUem=vIN>$x<;RI3Hgf zY(!Kezv*R>l&>)Nfu%fUI2nv_y@6mjaQQm#D3s?1 z6RBUs$?P*YkLYQfgI^{`5xvNGVWYg9>&%pQJwW~e3*^xS?L0$(r?lDhguKV?$7kqA z@g#NyI4{Kn7a^;`nZiAFGZ3=Ppm!=c>~;7qQo`?-ZbA}pLF_6O2G;PohoHWJC1OY%zBU7y6gXn-80ZJhw}_^pfeXkaPc0${ z&a3sEWxyT$54eYHk-nI270koV1Mkt#Lt0EkA%=WgP6qY^C50v%PSv?5B5#gALi_ zA`T?`5+4F(rQCq4j#ihOW~-=)YiXN?|G?YgQJ%(r z$NH$Dm@ceWE+71Zaqt8AdGJyPC* zauzZQ-U<1HZ}?uglzeWTkKE@EYP?!H%3j&=Y!4~CD0DK2U%6B4u0STBMggRDZN12Epe$E9 z@y*F}{yG9;2l=~57?Tv8WU`HxSMkR!6AQD%KGF#O6Ej8a?$^tg5D!_3Hu4TBScrF5 z7YX^~u7sV|y8Ke?mb3z0n)42iF~0^zGF428PKrbL*Fra@3AO^QNIsHVoBE54wG%mo zJI^f_R*1c%izLGK#!d%bP9FhkXHoQ_1|MsO-ot+r(Q!`2ak1&8Ia`7T+4 z%uv1Zvgjpfl_c6*g9FUJn9Jf>nFQQK2<%F9lQenO1k zKg{{$G@;PuWus!DB;gNA4=9zX=}Fx{tKVh0L9pm_>7jJnSE#I~(|lv_LHGn2WhxaI z8+nlrl4v__t4ByusnMydlv^iOAxad2$j~g9%C{yv+g#a!U_O?dpMg!}w}S%tL(4j>@xa@4@SQGwWeYry7Fix8PE9d4f6y_ggybRV3Du+mXX07IV^|=UL$kysX|4u2agM_bqn$r7fF;s+3SrV=CPyr z;pE9+d15W%6INsInIpVR)GzBjqXRXOO_E*dr2Hj2C6ca2#;b#b3UVp28N11#y0y}& z1QRk7pC%XhH{&^l-Q-%7-53&oAb9a~0@c+8J0HGmyw+n`d`mI%4k; z4l0({mf^C_Y)$JZe+4;N-%VZ|G}(gsvGRt5BW6@=X1vcg^ z)mU4koRnW%Mlnf-drEI_b$&hiL*8U@>4%sW;^{;PzbQ^2$h9imT+|5*fpW-W_A}@$ z!b7~+(TXWLS+boYe#$3oCuo{n4Iju3)2;YzcA|z;O03n?gF1RDYqs>Y4p1jbvE*xs zuBGri(w7{UkQBVid{TZ|ApEp-vu&P#A#Y|!$rG?qTc7y)nhT^a>`?1Fq7U&`~}YzRvXfzFC;&* z2K1n>(Vg5ed7;=$9Fsf)ZkEAf@!(-OS8l z6B7WQm|Npz`4L<-?iT#IOz-)`k1Ct(8>%Cfm&&SOQ7?#fBEs@qmt+afw{7ua0hu5_0S$<>9e_!HA!KSE3p`+zHvwxpk!2nC_h zY&B%CbW_j^gOh6qP8!DW&v7@M%Rf!tk3<-ow-Mn>UdmQRp5f*3T%lc=iqszX93RHh zbHe!MBo~YOacr2he^X3>4Ic)U@B_Vz`Flc71m#J8wjPTqM=Zs5Wi*qGlm$J@9K%jC zHe-2cOQ(nvvcX`WeD=E2>QMJ2Jp8$2Vae5 ztV@a7;zZ*dw39duyvmypcUkAXk<^f=LAI3?ZwwX!^E?;UHO4d9F7zy}ztR%FpR`ZW z^z)P$=qQkirt=5*;l61?Ci>K0o4@E|GRjM5_&#P->`4eko@{|7wn>M zk@Ab4gWCx}{K<^UMZK|^#vaN-z8yT!c!Ai9&s0)9Q;|K$KSV9HgLqRsojcJeif&?E zLLw;pXmka9Ah{KhXtHoU`Feqc{;ElNXdM6AyHDa{ua)8UiSPyMm&6jFFY^gH=C$#C z)IW(^$>zix+#7L{@YdYZvO^L4t&K;xAKn;drurp$3|!TViq8eyDh3XsARqFqB%VQ; zB1P1}LvU$Ava;L~2d&CzW;&493ayLWfh&U}kb_2S#`cCK!rA*B?bKErV20Gh2hU{)iW3TXI#5+-cB=C66(QYGe_@t$a9gUC3(z2PYf$}gl! zq7`h$M$ln|n6aJz#%&Wy=^gr2%BKqJXR&h0)4$bt(vV(+j+$`x8mLjDjG`6I(Z3MSYU`I*|5Vddw?iW6S zSH$kTR#+E`M=O(3d_%c!!585Yzt6mY94j30zoa{vZZjKTDX}%-GxwI7an(sD))hU3 z!EzUInUIRch^HYZTcXSsV(31}1Bu<}29QU;0oSuJV-4O&4}guzql6ApF*r!Bj{aIU z)2@^S+dI#BnOe{T_${d| zyBA-K_V;x`iv3oLE!*yWhiGUa`j9vV>7-H?$|CR|;9^6k!cKA`c?NDrJDUb< zxNoTgoj2kH%XWemdCxNa3sVAP(8BdbxD#?78I3JOPJ3&J3bq@ytr+6pzHU6&kNiq! z`KKb41Shn}4;UAjAHq$5^(Y{e5&nSD7Y_7ihh}G6bWB@lESqoSmkp%emKKl(Zb*npvZg=A@p~Z&%H`2H1qA<^{R~XhWf{{fea+i9G)J=GC(K>?l997C0mZHF!2|RR;0kcF z;2-n9*n7SQT?=VPUyAR|)fTqnO{W{L?GcSs_3$TEm#ixo9--H#Wqo>sILV+2P(@)X`eajnUB1oQ{>8GT+9T1HE@;pSRPWM zy<)BD=KE-WO9{8WavMtu@{NDSzv8FDt-XEGg=~7!J}iY9WQOA^^P6(g(51my`NNP> zxIX#_-^K`e$AvilH)4bLf*57Qm|33p{8S^FG!rrLa)I@^gMWqhZwcU zm;DF!gj>AzFL{GBklk%b@gxd|aw9Mxi6&6c$#5RX&oqDYHR5o{g|0`W_$Ay1b}c_; z^$))c1FX|Df)6D>AiBU0n9hL*$acV2Fp+rU+3021o0ttLB9B|1$gEHiY#_V?ON9eS zGqk|}5w;t3$OwKa_)2UI6MDDo_W`yvzV8bMV8b4_i#_FR-OaiY5Ipn**?g1 ztdHjxv+Dg|$@N%S1NIx_M5CrS)ISm#B$JyqSn7Ja7xYW6DzDUc72C@n`0BDwt)U97rqRJMbmq+JN_EiZS0o(rkaP2?DKtE{j0;<`Yc=mtd} zdC2+|fs-f6z4MBJv_eY0gJ}S6K8G|lUavL>OolJAqxq}IK6*rcgloxLfgOqtEU4=2UR=CXH)prZ^`We1E<%TCD+$c4Rd)Y1|N!O$MXFH)o$#=kEWp=3< zcrN{e9i6`z^xFmjLyal4$vc91YtBs?<7o;P7)F?H__uTA(KDuBgus`*K$_= z4|X5iAKoKA@s#slg}TGPiVu)@=}&{=&o4vlHNXx9wa|*@KmnT*Dg2KuZ&@wq_$EnV z;{dQWG#yDp_oBDC_@Xq^A=Ad>i_jCgCZEPLQJ8b)_bRgZUeU(nJn)wIU24qbGlTVA z^=a%(%YI<0xiNS#Im_I#R!0u5^}|x=9p$UTFY(;UZwaNLZvV%mSr(bk@c>{Gaa75J zr0J&YqN&i;_;lI=G_^d)6M*Z$b;KtceVouK@1^AxJ5!hse>NT_=)!8-`_O86Y zmP=j;Oa#_tk3pV`RUp{EF4<}N2u@=caxuPv9>2MiuqRDlUlmQZ{#4qcj|2)UVr-bJ zte3XpbaLys9eP*JXMeS%-|+hd>7F#UHE>NoOn*a)0~U&pIH&2)Onm(#;v{yR8w2+* zDmD9%7P`*#e{{P|&*2*4g1|FmqL6KDnKd1vq5W(_Bu5vwa++nWJYUG5+hXmQv77%k znSJG;5y&s%FYHn5KHbyb*Ib%g$oyo|nWLVjQZl3CA^i(T@V{iWd_PL^Ti6*`>!d{D zsc${mL|Bk~O?LrtnEm>0p3a*tk!`#e>2z+JCulhgG)G>s&y&06o`xELgrTwftYZCzGrCBA8r0K!s_;-2pEs8t z9_RG$#y)Z`wwwr~dm!@Fo5-bXu8%9OYkOW6=t5tQRjk+kp-0MXWZw-PkWT zPCm~!!jJQV_$WKs+o!O3c$oCu*U>VEUrjfL8qf>)K*4zKQ$c$CtBsq8-a_B_w#Y8# zC36c{$M-~r!?l>#;NG|`^kQrv_$ui?Q35~YRF-sXl$%~Qm2ZPTizE33Gz1R^+vJ@~ zEc4uYdH~gtEws1rhrt_DEF8hFn*;nq=)Kw)evfQGC+Nqa3FerhrN9W!K;#K>8B60@ zntS43@oOeGx(u3SsziDW=f#VKf7EmV6#_MwxqN*s2-Zf{@=da?p%X1zg$t0w9QIt( zouMZt??iHdZX9EoB(UsOq<7^S+*zZI@js01=!@~Y%6tJGu=19( zmMh{?Xr&ZHXNsNkZ@>j;a{goFa^V}M1@IXDhS1<7N0eX4)!ige!pPs>PVsC<|$ zZ|<#(!%ESrNCcfl7b6Sl0mfN)H_vnSAGR8kE41g26prPp@-d(f5e>Jo3D84JPR$MY zCajVr(R0$Xh?WI5w!D`r+`&wxRj$}PSa_xD0Ho!=2h-5;$S_X_G}%`Np2IC-w
    -fG4-WMxwj@1#lWVPrS>&5Dx_Y^}p1QEMxXh
    zR=NYtqxYijtS!b}@)l+X)I_<;O)hLv?i)G<`5VyrZ-UF&PGSRoQN>Qe3A8EnNB(%k
    z$!zq@Ma?)5cd{%ot}^rlE)`sfO=rREV%`F^_kV+LCmd&SW|96Eu3Rh$iMjE@W$?Xm
    zTyk&T1vQqI84R8)NHz0MsJ*GZf3mMGuz>$3pDG*&98BoX)YIp%Uy7nYdI3=|$a6cE
    z2TsCoO^=z@*fMy2!CSbums?&Zb_tt@@mw^{A0-=PF;QXXG(ZzNLGEG~{P{1+z
    zLgG$~C#eJcMH&+OwahWl!lLj;pjo;H=I!2I!179E&;@C2uHbuduEhn=IrFK=UM(|sTbDW7%@Eb{^Y;W{XL6KM_Aa@J*ixFf$px}
    z&z-aMH0$WEoSR*Y^%GkF)qP_;XOOLigMJ0EgBLeF&Z)xpFtowXj!Gsmy(
    z4z;AGaVzqd-+Fj~kMv(Q!4^M!uUrAR#3%4h%V_u*Gzeae{AU~tWrNwCRcJXOp8sgt
    zpWie0W^B8ni%@<26n`&&L$(UH0r=*9C$u%S@VzX!$4&QBU~}Y8pqHGjB08HtnG^=+De__-5=KZUwNJxkL->ZSh9}AcPqk{jSzr@P+@a
    zxvOda`s|``UL}7^t_CuS?P6+ZSeV`3H#nKhKf;waM!m(ByKI)_H2acywQ)8$5jcTf
    zwv6)h@jJ@-m{$4gpodHa?o?4_nDf^*-!N^29ocu8`uV$n3TQI;&|)dm&EK-W8UN(s
    zEN$ow;w4_pt7Yhm%!A&5?SLJoe+x|J@xGkgi{Jxh6y*w?U9Y8O(LT{mRBcyj3hd|(HV>zU4z3d(3sF058ZHWEKFhP6PlFOsPaH>ZTtuwPG_{?yFG2pWJ6v1w=i$?8o#Mx-t>mS4
    ztE-i*h-mE8xg|l*|z{fV%f&yOXpeZpUDF3wLX`6Aalbj=r^T)
    z+a#@g+4jU2F@ZCwt)%(z5LcPD8EUV-x9@S7t$(}wC?WrzNcYH#@Gr_uIV+`uZMEtP
    zepAK@yR^OJpVVe*vjetIa4ombcGL_Qqhsu}wKdYpvD&`enPL5s(!#MIFq}L~&z8>Q
    zrbUvJjo{1h;D}w_pfq+ngI~f|!iR%TBB!}!)GJq8)X3M27>Pa+R?#f)g6)L;)DLA(
    z=x=7P?6T%rr-$A}K1HxlRjL{BF6E7q75PJ%=E{N60(0veL0#<7(5UFPAY!ap9FW$?
    z4WwNujp$>JTijrIcc6mfoZB545t?ehV8=>o0YScpqpfQ-`8;~kc_FB2m2KJhcQOkz
    zI3Fu+A&i{R8LcWpKsH5bnNVe81;v)x9^N#?mNT$|V!`_gdIt~UqfKxPG@j7Su$QCi4{OSHB3bIR`tEpK^
    zY;YtxoLUzY*d1!wE`-BnUrE+QazcNR6Rmp34EP)}-?rPbOs(jck$O^_7(FYVDWv6=
    z)G_~2jn-C&evQ-*Ey2o%+9YEsy#vdIQSg^Q{lI+Qr7eq`3hxZHbi^oQ>tqn?n3PCA
    zmr={MXKHutGwtc66RkUKgB_8`?Vy3Qkn@1=|3}eThNrE*e|%eKUA@HuNeB=H+RokG
    z-QC^Sxx3Yi6Ci;kBtSbob35B_dgkt~XYRi9&;MOtv=yI!9CrZOPp4Z=f%vj$0O2AIM#ZE(1nkZ-SY^Z0li6Zsd%7_
    zy3Njx&bMqdOVJ*9MY13LDWd`Ni`*JHY^~}&6Lpvm>%Ty8+@MHZw2I)NUotTJM66w!
    zqZ92Ks$J}Q1SV>wRHt@jHS}(`rjdgqBzp|4j1Gl7nk|~O$a|{Dj|bY8B?YbQJN*Ld
    zjo@TcniTdY(6doV{ga3qMhKMkYDtcHwef1n|Hw?LAh~7M?$9QgYaN5`HrlW|;@`nL
    zoFDvRh%&v|@4}SuoY-U2-m<1yugS@_2zZ=WPF%7+cEjPO_=j+7sW^5EH4r)I&`6y5
    zhV(z;-_SM3Y-)|6m3ar%o4*+QAzhM|qVuRc>5;j+lSplg%;p2)b?~v~dudAKu<)Ij
    z6=N9+od}R-edrtdg|5w?HnEZ^I3v2&_>Z+hv|;HBZhh<%nHn5RjSEVVj(9vaJNv;#
    z(usy6k=muZhzk7F*wgHbVpHNgTEDbRdWJrh*``G5b=m*Sb(uq1Vpbk|)B0VIB!m75
    zS%^*c^yF??8|$m$4XmrkKNQ7qtufU)s>v1y7U%=hqPmUrUx&0WZ4WHP0W^@pbg
    zs-w;L1ma5Berv^GyFe=8F;>m&Mn9yMp=YH>^d902zC-FQcEP$yw&+kQ3v>Fzo)OmE
    z&Ap-D#SFF=s};Fa<_Yfzl!MpAF6oQN9sCl@bHgsz#o!4xSu9sHB-TFqKKj#j8h?Ye
    z*bMX8gwft6rZ;@X>Iq)ViJb^sw!9J4*fPemDLK3r>m2M99gLPUPsjJhYL-@s91*XQ
    z7$b6Htu6E-`3^r6>}Rbiq)Cb9+S-FS<9Wgk|s+Ntdl16;xov0&1WmYW9C+i4%!=28T!nWD`m+lVh!tLVE-VOm?$0mTl)Q#!&}n-qwjrO&Y(JURSYEatX}BlR1k
    z0mMXWPV^?XFtRCp-Ta(=3yst{ESJqE?ECnmtTQ$leIZpU8z4#H$C(qvW=2~6mUr-j
    zd`w&xu30)8%U69XE3$MJUT4-3?z?*98POsXaZj<{$x1+b@B{HN(rMI-Dg)WEdZrm=
    z+n}V#De4-hu`Y_O5dT$9r83Cj`tyb;Ihc4xEDW=rYgC@;-&i+;3+h0f($#bvGLxlW
    z(VN^P(++wibpdS?njmLtG9$Xk5p=Hj*5BCpiyOrMostk4Pi>7lqQl8q+|=k1L2Jnu
    zpy)?xtW;6D9qow>COU+dh>3x)Ffp{zY?m_0M^cnr?&HkQxUT3%@r=@D-Oimezcr69
    zD;FFi!p80q*wi7?z)V>hl`5GvCP3044Fv@rAA0R8Bx<=bEhh4k{ay>%Rl4+tg+5%9b}%v%`Y1s38Xzm
    z{;>`)EVHH%OUWioTda~+m)#}y6RAs-geC^T=zVLo*kiG?_90%IpB3&vdaa_hzNI~R
    z72QMLg4K8g#dSs3_FD0qW
    zQRpsDTYg}8yttKB3#@b%8-sSRDpc*ts%EwqR%CCm{=v3Wdh;eMn@F{cG@UTjE!B`E
    zScuX`TtU*4PR!&=BYUh(V-1AmW=b4b+D#mq<&aK_ul4WJCL&EO?ZO{PKbOQ!jIPQw
    zSz4Agr*cqxaMfnqu+TcwIy19d_Gnw+k$GgKGS@sZFgn}thngM(BpsiIj%GRtzfBuqw)C6DZ(hjHg6COV
    zi-`rx%(KM7{0}kRy02`fvI)9}O~NONdqsr*YHbj>lbIKr7kd;b2oH?3!nTEO(17uD
    z)|9eed@nApbgO@Rq(QVPHja4;H~XG-lDKx_}L7o*mL>~3Lha2U`qW>@avbIHfnXrLU@P+yrV31?6l+9}Z*{Nl0;
    zkt60a!5g}mbrHIR1>EbXcBL!LBce;Wn}+AX>TH_L4q(A=--D6Q&HSQ9iO$z^rYkIec88EYcA@K-JvcrLfU@!F&VV$6t#a%PbJX{78dNnuNs_?~u&W1pYkNiFy;+ba%kYok
    zdAO5l7hc|KEgWt-TlPsRAbZM-GdX+eb@vlEc7*W_o&P|U7YedT2uTN%!v#p
    zr^$JmG5iXC0eLopliSgqvAXnf>=a^Vk}bbWsnR;q!`MospQQmZ-8duEuB?^et$50+
    z6;sU>*#4GM;TuxZ@zA1$PY0g@|Kn>%Y|$d?ecwE=QLGocH2WX9kGI7Vi0b4rOD=QN
    z@iPAf;tzB$E#kXTC5G9tOR-)29nBuNys4pOO5|_rbAga*g4M{*Y&Uox2x4E&;YcFE
    z@KN6z@pg<1C&vzQ?J}2`IlK+FN%~HL(sXuP=?-Xq_Sx7gY?^q<^h^Zt%*@5~gxEvk
    zW9g$`seT&2(w%eO#)fQYx|
    zZdg}NE6u^GiLIfZOjmqj*(-n`CW|5C2zV6N0dEv-8M$g^(2eF}%wTZ`bDmlOmpC%D
    zdUT2B5e@pL#m-=g&{CpjXcGAVs!BXTTS^_YF61-vEszGzw#?;*2&ttFq{9vtnMx)3
    znnGteiAafVL5!WQFfT{^j(_$Hb7!Bz(%azE_Qrt!v6=q+k)pr(0w
    z=((dk7ejh+&%?LHjpl4F2ayOb)jHfP_K(=tx(!o?>eD;JwWZTOlQG|#Ngj~;OJ|@O
    zp}KG@&w4+@u4GYZkvJ8*M^rQaXTr>X$Pji~IECjjN^()~Jozbkb9iv1zU5(YZsx0K
    zRqI;xh?HQ>4vi)+lKt>7c`>Y{elQhevS7R6OxS|{2PgRJ!dszw)MRO^{2p-!(?;x$
    z`)JQdKMRZG2nFOa3bMqTu9AZd2JUjS2XPktf`NE5vb&sdT*dbn^7sLndT~88%lI>J
    z)=_3vgj*7jV+Fn>RwXhGoxs!V2I^`INH#ZwT6?
    z2{l5dM~pN2URoG?4KEQJ#Lf_qB@~&D-3S-TPaBNrMz*zeHn#HLAZA40pyk37{8NLS
    z&CfzFO(R5erd_++Tv=*F6-P3Iu}oa3SgP`U6o(?65pYv8e;2M;?4R8*@=hujo)}4u
    zbmTg7$y8tK4>w_*N?Onrkx^p**tS5i=^Zg9Qju(K`mEk2H88H0Hj=5z=~Oo|52$Ev
    zq64H)#T3-x=u9|B^vkW=`V(D-tLd}$k%|rL(th#{E+-%Vp&e?pITy55H$1GPu
    z*;ud0J8_`}jC>@mu`PbR?8R!uo|F67DR^!yff-`PBUKDrOPfX7hnCw?mf3mc3#$#${5&~&VGsox;8oNBu9HoO*noo}RCsMmGXBFYgV)@`+SUWdvftB+nN(fpLNg
    zMuD`^XuNZrX0Rjt2Vk8k0F^!-IuIBJ3a;k4Dex$mcg^xuQWt}bnFwuiZ?JDo$%p>i
    zIMm%(b2v~*C%C5gCjhvY_qV5;C_f}TE(oegWiMLHTbDl#a22=}T}rz8HTijp;mB%u
    z*G3*{$*`_L-ug0cFaZ{{iBL7;0^bHY4}1>vV%FGdDfOPC8O>I|^be<>+81Vw2d>){
    znm16K(WQ8<+2lT1?DwYS-&2p*A
    z1d5b%Rv*txLjKU}H?&e$vS-0|+U)zE?z`uKBE@~6+8EU*?V;Rrs&u;4wu$!Xa=-za
    z21yN}vw(&@riuX;@2hH`=%w0AKzy~$n$v+&U|28{{s`jxy-_SSHHHS_D&!+4uQEtrAjzvy^sevW=yOgh7wSdRkVLBA(
    zm^v7IoF4((0R7ySigUpU8u`wGs_EFmwgj%-?G=HU%CXwo^clqq#Ww$HXhyOUss`4|
    zcnVIJDg12#i?5BdeY&aWP&C;a)OB~KIA$j8&qH}ejQopHBub@M*eZ%;RQJb|@v#Fc2I
    zTtAr<)hlh1M+4?Clu80as#XB&X$%f^mujy2t^;KmPFEjrjXSC-_P(~61A~=zx`ykb
    zZn6C_y$HDD@+k6|-SMA;Gr?uW-4k;FGn-M=sKP#aNipp|oN2{~?Zq=^EKODV$n}NFazDZkMcQXz_Q{9Jj@9gW9Lwx6a
    zLtObjujX&$rNf>uH)Fc4hih`iDgBVZ)WRd+ZTB?Sra+qGQeIO!OJCXZR>L|xU=8Pf
    z-GXWj+}VX270V0t8b&)n3HjmlA8@=AuYA;X)A_i#hi09Aow`P+HK=DRsysC%#mbN{#Q?V;O+1Dtev8F|!4vqN+&`WHt_~kTQ
    zSf;&hU!&enZ(jFD-4O2WHh>519GJljQXSgRn(nAAv0ZW3OIq!23(fQ01ZeM1JD;8?
    zXDMQ~N%kwn%RP>wWzb&5l9HYYFg(Igz!vA_s785y7A|(RMmYD1z!23)iU8`k{9h^FK{_|L(jox_DoeH6C?sMMzs-XLCASb#%B~U*GD+d&fK8
    zc&mH+fM*Lk`~ULI^t1qd3Z?8`Ty1>m9|D%zCwYrpuk-`dHP)G(w-x_r+sB)jl}Sre
    zO$mE+IVHm@#!A*|0q}|Y2E1Fjn!X7B@!*N2Hqdv#+sd=TvYE~+=vEk171OV@eu#BF
    z1SU9Z`bMYsrXQtF3K!E|G;YHbK%ss_*HUoH4iszs|GJmK@!B)+FtD#qP8|bV^E;;L
    zU2i?39kzn_M9_{vf9!MJ60Lz_&eN_$_hH>RSFQ>Pw)gIIZg3v;)^}A@Bm%!wS-y}e
    zqOGn0(m%Qa^ftp$&n!rx8R9)uJkHfZd(C}MUZHH!yc&41pP%Bh~>#ER-W-kS7Wl^dE2{H2(y%>mOZMzy_t
    zvk>{(X!CTjRF!{%cZQs8pAJ4`nycD)jzb%?6MY%%B~N39@9MxTq%ECb5`wf%`^ZY=Rd8h=$#c`U*8UJd4rDn
    zx@IXS%P;UPcif~OX|-BN)f{MG@8{_YElAnqU%3VVMif~WYY@oR7ZoBS-{jhJgwpMO8V1PXid8zxPX|39t
    zR>Qm6=S=xvTcC3rPPopd*0HIa%|X6UmmG9}t}xtK3p-D+gm)n`6z)vVrSJF}I{Jbw
    z6x@c(YkLHu`V4i?k}4&&V9>Wi^$AK~cDQR^4o
    z^DLI)>$Ovd|B?3w_|E6XaKFx-{a2)W@uD&2U}dKPS&-Sx*NC->MH0*xPJl4yb^WClJ6T9
    z(cA1fnjwCU-dJI>FG@dmMArkLA2QQ-oP;~*&Cw*-dTFcCRq`7$7l4i04Zi*UQSPda
    zE{;u5d$}81x4c=}$^gx5rqB7ZHIEeOz;u6q@PDqs@C`CxI|z@|ywFJC$Up)*F!-OE
    z@~loC;K@io3>}ewrz5!EIZ4?PG=pt)Pmx7nn~b9wBa{a;pFNQKat5v$oqT}yl+<>e
    zaGcPf_K}*Zp7WX-9wi*lpH`^V$DtExOMHmtzm%jjnRjp|c(*u@(DOB{yI
    zzD0^fippS_Z@r&$or8zc9sR$+oTP=zI};Y)FVzBPf4Cm&c1{K+uFKROrW0tNrc47s
    zKYab{It1~3x3yljaD_r&v*@@=^w_NcU}i%u
    zyk5@PmOura0;#kzzQ+1K#h2Y5=t@9;?^0!+LZ-x^0w0lFsvV_mnz~Dopco1)^Dsq!
    z!5!57Qs(_T2~puAy=_HOkv6F_@C$
    z%0yVrV&%$}*VgXyiNJ9%KXJOgsjpJe;-m*@8plExivH{R17{>}@mJR*Xr>kW06I0^
    zRXgRP_JO{)yBOHy1K=T!DR8wwf@?K+9xm@6rkO-Hqt7Y+gEpsDRo>FyH%#|!Q4FN3
    zCO`Hz(slBy=_<~CfS3Ij7@7Vw?Nw+ScGidqu}fm#Z+r<#3f>SGVYd_D0eM&j$rkxW-ope5&PBJInWf
    zoGK3P>4cz1zDv&j8Pim_cL0F6HWeN7KX7mL*OwDzt+ZM6PVIoSJB9bO*|rvi_X6$V
    zo9+(UG!MoOfr}kxDI0i7ue06po>8>P?UPoq;6F!b-7inN;tIIRl|(P{5Xx*@4an+e
    z{aw=nfjvM|r^O!x8hR?FFLGXU73=PS_iY2$A6G8}E<;~6H`LeDQBOxrea}F;CLQP9
    zLwAS1(_^3n7f=6{7KK7cu6Kz48|$+#1zX$u(TR@L^b6S$o2S9jbeet87jU0uyrZ1|
    zbLyqSAF7$!kM34JIYq6ju~rQiyIZ5DJ+9=`iYveJeZ?d7LjQ8@N#JZ+Nc&DVjjlq!Pu$~c!v!=}(XO_H*f<}XDSuwOe1XrRUEbsM_cGTi-B+`0&O
    zg|6>iLJ+?#UH#4ypKE>@ZNt;eW7S4FbI$hJ#feo^i)pI_D*$NPIcKH
    z+H!0#*vPdJs-es&d7sz8o#i}dF9%N1D3i5bN&_d_mFB`uPIK`+ZB6wOWm=U~6$ot5
    zW_XS|Dm&VP^X(rC%QU8pXW&-XhN72rMbB%6zgSYAfyT&0cdm1uTnnGOW~Elm&j8=0
    zU)D7Ck1m|*QWV`(K1xFrchl;C(^l0}Iut?sWcvc;+zh}z%ylhb*81tAnwEARF40W)
    zjdmZk-*gP~wANg-#j7}tS^LArx_r4iKG?mG{FCEiu8WAeBD9(O~x$#p_A$F()NGGa?;?e1akRT9uH_q6rNuh^~^
    z`3F<_GT-xFD=~LN-F)>)@P#HOuPhr_FGZ&|_!I@!yla};w#|ym8}2Epd#ge9;|H#u
    zsotd9t=tfIAKaw52EDA%CwVcv$TNptkpEk~Q5$tDZT*V2E7s6ON?+0YJW7`>^W?RF
    zHIBim4X(ag5p-x*+1I-|m!v8;=jPMxXH$tm8V?Bl(dXi%3+1ASwh6E$lT<21)~KE*pvSy7JXZ}&o1OW$L4L#Q{91iV;Q
    z2YhXBmC=-r(!*U(k~Vpa>678Qz(-9x-7feFFe&ejj--2N|IknDb6mA`$Mo%2T}_>;
    z`JyqTR&iH`g0{Xub=NZ61-rq&JQcEkw6RHF9R(%7eWRLyU)S9*(DY`F_8U$EasEN0@58$$!HLO
    zg#O4HiLkDa#!`K`!QrtntNsnvG_odqaa}*dShA>ekdxFO=zD^{
    zTCBFl%0DLOL^O^|_;&OG_D!m&n~JK%%CT$shuBLlB)lq`$}bfE;l2o~h4I>UOsPD{
    zK)rbgIR(x$n~}->Z^FuG6{Xi-f?m=r#Pj)~?6}Bg<5g*jz9YSeP7@76GkpW=W2}t5
    z$^Ok&!L~s&XrAgKp`a}|!a0|ps*I(#iCMUs=r{4UcD1>loM->U_Misy5n~S6j2%h!
    zB|$VHT7hVTeKkSQPLvP7E~7I0lOL(ep{1FBLPNL${xIn>SgBE{jW2-a;}5bH!4!N7
    zU&1y)%adKew$@!ngVY{mPFMh58+4Ue9L~J4Bz5^Wae}&Nh|ahygyn|
    z_>T*5Yxy7O?#v&IS36K#%R5YOGavZp$QHp9Zj-8A@TGB>zKrzA`SKeD5snhA~%r=p2tJjbTL@C!bX0SdCuMwJX|N13UeR~{tmk1M0
    zq!ROgL_(GtB2maaH?39D4}YkfYhDCBMb0Oh>Xr8kr{=FQuL2%E)U%
    zC-DMC8EvGB+zcK^NDYk5H&#NEO+xqvnFa|$b#p3tBm~D68&~nQVttT1{6*%V;kLCz
    zBuXNgaoCmUpvYux4eJfd^YAm%+2B0rE@vbu;*Z&=%ZY9yb3~)7Gj$Am6XC3fj9>Un
    zqh$EO$_^X$FOxz%fMpopm0hs(iPYv_VBzpJw3h|qFGnxp7H&MIV15wejM+Frw1y#M
    zOw|hFR{lP@5r2ZFU<~t@YEGz6EQA%h4g8X8cMNm+Hiha~Uy^-!9FfZU^6+=2>~o8~CIF;66qLbgZf}
    z*4*-ynJRa-m$~II82;cMhYi=$p=MB+`wygvhUQPPTJ9)0*VG5O?*GSUmmbsk>gkk-
    zuVNQqojkWFfVhm`GnTnabwz=3A-go1YsFPF{*jrrd3e0Lh50?YB=R&eB-k99VExLS
    z4Gbj48yAQd@hygv!F$4lV1}g8?B|!89vU*D`sy2G+Iyi`Ld>CN3bj$1s%au17R`*b
    z6u$_Nct~E!&82f8kQ>Bp5(isK6>UAQ$U9o
    z;za#SvI^0Rco2Ap{1l4!=?o(hJsEjmRcRb1%
    zKZrUXXwZX?ic_UAyb$^aeJJ})ej6ICw+M|bS<$Ar#v0H3r=$FJOwEjQ#02QLfi?7F
    zi(N;}+mR;G<(a$jm0T6>T==oE8A}P7*j%g{*F2oA|6gd8;jGjg+ZE~sm1EcG-wF4z
    z@kk}pc6D)}A5oKko8FZCE&8IrBP*!YWj#gM;t-z&zdEmr0r41tfnS1s!beSA01(N%4d21TJ~|{k*f3-k~dMz%uq+yO)i7XFjf#6
    zgz^Fh@dV>YugmnwIz6BOXSw>zj>p!Vk~nX;87?x_B)>rukfBK5sO&9a7DIoqEYIYq
    zXjG9ygt5Mzu?*oWNRk8}pIzwJ3iZgU`Nye>XoAoe!+gn>RoF3dHa|9Y+E609zT&h^
    z!voM+#Bl3u{x&}eJ*to2Ttn#0XMye5YEt)&HN5erg>0+P7S3FOSw0Enm#1Ci*3-~cpWS#}6r~vo4^ac3Kmjs6
    z+>XyfIteGN9hjd&LuN1D9qmk3r5ck3KzF{K@Pdej=6f@+k=X%iW^^6)1@jvd6>sQx
    zDbbna{zEEIr&JWWAX_!#W2ZbX$ra%qQVV?n0fi^xQ&6P1{$JBA6ZiqVir1P>GnrDV
    zT!j<3M%XVzL+nD&qbjH+i4xz+RAjJNpcY)HbXO
    zwNIL8NOj$_UNYv8oyd~RW@3r4J)`0enEoj1aZQcod^Nbrx|&>fW;qKP&tp%VHLY(M
    zkP2oSF=Z@l+3(vb{xR*S-GHenpH8k9l%WaHnuex9nDVA>#Uim|vURI@fJ0|ezll^+
    z7vwD0G5izXk9oySTxalEa2@GJ?7p>VIo^@K(zlDsgu7Ac#40S;{vSHV-rK)PBt=>n
    zF1~WT#m`e8!g%Zoo`Ngb-RM8&fnqr%-QHK668ej+V5|`I2v@l&5!Z-#pm)pLh{rU!&|Th77Onr7jezd!^mKNF*+Psl=TojP9~#oh={)>
    zu0sYy{xkip-6`x~R*RR&Z1)vmi@qx#GQZ{Pq9^?Ojq$<}_iS@Zwk}d4C19U56;X))
    zNsVBZM_z;^{<8IsFU*b)RO8!3n(%j!NobmX1UfJDfw)7pkp2zCP%*MxDhk`pqe8z0
    zSdZ7JinI|Uu#$WsE|)EuJAe$Sks(B`79CjC=z8?7Wwj|uT4}Bwu7ghJqQoX(3^53M
    zCbp6pHyd1xOhNkktA>YKzfe~>1e^u0jr<1|hoLA|2GDBF){z2aeNs-Fp;r8k3Ht5+Kr%f?(cVn%OBHFPg%Bx!Pahzz@n0;cbwl`-
    zREv6tPP4|M0SxALqDfS`w3dH_n56ntYvUH_KC}=WO05nrcO1{ENY9Qep+<_ch^V!-
    zyl3CSG}8Q)d@0q;h>;_>RpJzKsctb1NnK)&7
    zOD;A4;b~KaSS|jpbl;pty~Q7*Qwgh~3((aL>StOi)8(Ye<1>Q6zQ5s$(uQ@W`ywL{m|To9RVF2#ByLfSTRjwM%kCZ1AO*7xwlqZ{~!89KBD
    zxeC{bAz_bsSilp$ge_ziaiiQT$e4N%F^-B!kRAxt#MC7U=O&>%XDW9OTTITj$OZvE
    zNSBg*$l7*?YpHU-=@m6J)}Aa3w-22RotL^u_sOVO6&o*XvrGo}ue$#wPHd^r
    zR@@MNO%6n_!>_rJf1$Br!3KyFst>U&c0Q<;x>GfT0--lITN;7oioemt
    z%sh8Fav#xD@dT~yY)b4zy9HGi7iEVA>JJAWKt<}2SR=A5(vTSzn1Z>o3b?`KEa{KD
    z?pOF*8WW;%QQ3+v(}~TpFe=Ho(o-Wd*_$BUp^m6daZy``&@rehp1_==?e40^I#O4^
    zlUyJ9j4ee!@c-i_OMSxyc~;gS(}?Ioe4c4{C``Y?hV$c0fU&Dsep3bPhH#x4ZFX8@|Sj(AN?l8BRxFd<|AtdM4D3
    zH53Vd1HK|U)^iW3W;wEKjWEx1OKf6IWRltH;$LK^lHTs~l3=JOwX(d$`V%9W{%Nfu
    zX-E>hfJz~EGqs3ifyEqX3ao1H(_nd=LX7jQCwP1x*9F@~eXX6
    z;t6sNyI)Q`)%4Z?SzxiW-OygR#g~u%PJYWT1oYdS)`{$Kj*dh_C#;0~IlrD*AP$FS
    zaEA?5g->V&DI{^!8N*PxXXgLVkf4r@j%6Z~LKM-{Iv>m7_OJ;-Ti|~Qlld&D4RIx0
    z%2abMLlAK#Ucvs@@K>k>*95Ou5>I>x?K3tb9e5t!4tZ!(GYy0DkR&3(N3siSx9~)n
    zMCPTz_~BRy|47af9U~vnHKodKn^@%EXI>mOGQ4&fxrDEZ3*0iKeZ-4SEh~rk_)M{P
    z_OXxK5fp`?}$_?bc5*IduVDH92MO7jNp~_oPwQ09ntQ
    zNraH(+{(g9#
    zeGs#8oB^|N5<)^aGiAmIyhtmNAla@Hm)({M{vK{7`Mi8!vsU+pksJ<YZE%d=Qi7FQ!C>=py@MKu4X=wjO-a^+yA)N>E8TjH4^lslT;Uzzpph7jsIMHUC
    zf#zW;`s+v^^fsprU#A;}ni*S=HA0v(gM02?W2z(Xb8I&dzN2tSU_Iq#X!jO>Z|-xb
    z5x*XONk_bT@N+0TumI^|x?=ppv#37Y7u+W}WCG$WdBf{;>q2d~TcLU3Dze>88(M0(
    zqU(;lbba@K5{%XS4bhjA|Ki%%0=;~g0F;`g0hiDc{+J|4mF+Ci1%!#87u
    zznE$5x?>t==`TNP_F=#I3fL)MeQ#8@k5(hcV*)=UILL536fB*_ry)H|Ke#%^zM)ar
    zH10JyQv2Od-DB|WX4>IPjJWoWu?hEQOGww9-5Tg_o~xTxfz6lw-DDUkBI7qsU`@eW
    z%zXSTg0jW2f6>)sJ?aj=$1<7)_}ju&GET}tyNJErTJETK!R9he9M+2J2u^}qV-?F^
    zt^1moEG2E;jI|2f4Sfsx3^m~SP(P+AQbBvc^P8WeU+7(gT+x3;w*W!;?en~ADwAVK
    zcAJsJ-FLN0ZoF%^X}$ZGzAH7E`y8x|USvk1pXB}5C&qy5IJ{MuYPwGP(4fa|{M(;z
    z4Eg$kspd84V04!8u3W2Sw0S
    z@L2dP)J^Y{K%2sc`R}58Lch6H_{NN9=nbX>j0-n6Wb^5$mQ~XO>_>21Is`pumcdkL
    zj-ed?KY9kel+G}@gH_pNe+%DYa;uy3p*KaVtp4PvEgz``Vi5o!VGT$9wmjK1?|gh0%BY*XfM7d_&>A<8xWp`
    z%}j=Ln&~6-fbVABvx0K7{*;)bxy^P`zC{46jPD=L4xckn%tdq&AK`}KJDGz}BmF_)
    zD|?v$O34k437c^^)(v~d)hpgAA>0abCisLODE^kpfmdNfwg??al1v9f93M2ywYFk5d4j|#VxKl5dr}X>5%9gQ
    z6EuO$K_2MUXdBHg{g-e8QXfB4f;$`HJtRcGoXaC;a--?fYejsal7uUf-~5H>i
    z+OrHh&pDC1r5m-A#U{bNSjFfQ=m_y#PH3y~RmR4}>y?MNd-}V&O;a{
    zR`l>If|up`kV&VTnin@Ry-!JEdz1f~hT;HXW51X+8K*e`j2f>hOymw0WV)l%`3rn1
    zV-G{xrG^2<&7=j9LJ0kay9iuYbP9GQP6y60Elkb)SJ`4dN9^p_W0c*qJc`E~hcg;+
    zn9+jI59MQ1i^nj>tT(1KnZnh=tZLD)DH2(92+5#^Xpp=+ii;je)h
    zQ5tQ5-HR7dWFX&&U9GiA^K>|GkeW#+4L2BA*e3h`
    zE^?QG6kbNP+2TM;$$ox|{x)`*Y!XUD<^!e1NnmZN24v6|GtGe)w(8+txu1iRxk9>!
    ze*u!i7GTN3{Xm-Wjr%Bbk$r7g%FW3h6r4`p#_k(4eG`#$WEDTFCb^2j)D5$^{o*fS
    zpP)CrFn9#Wb;GdUbr*Hh2(yw|?t6v#3(qjiwzkCknEIfzP3OFzezWnA>Z0Z;zLw86
    z1)(*9-*`W?E+BCyvA4!8n<_KgLD)9P{~Mo(X&S#QYrz9kc`iJ$wvB6iN8RBDh2eF*1#g5Cmj8%4z+0ljzhsTf)f{Q{IkoBky
    z8iAiDWS=~d9jqR#5S-%Qn&~h+)WwCJ%uKomL{vKLctX4n-Z%F3XCl2}TpmT=Kg8O$nFfg*QD1rnuhSL-%fvF-$AqGNp*7G3dMo6UZOyIt
    zp}uaap@}56UsVk|OOWJjF^yfzNitZY(L4E$+*5Yfw(jbVs$sHAd>B8Gu4Gy(ScD#Y
    zLBU{Cefj6<=rMH)*aCqN*!ZDZL*aXjPUf?(H
    zh!p)YgrR-LiUA*4?9kgpv1)<0CeIXr_oh@R&Da}EP9`0`1cX*>4L_=s6;jFSiU6u}4mI$=e)
    z=CZ}izg$n0qCXn&Rc*v6!RgxG!Rqpr3?p7-9aoQB>WML9@ZR!U&U8(QZU2_L;ePrS
    zg+_g3S!PIUo=T?(i*$BAk$uJ9FCvhqu4&v|dlgbPf|>{54|1O1LZs41NV*^5tg60&H}V+y|y-
    z8hxeGv5#0->rh^3YBQB7q52_lB~{7J6njBBSS|U1HgAhkE-&JyC_7*~AkkB*7h1fn
    z((n9Vf>CGz-TH2MgOVyG)`$2U!gIKkrr2zdc`z&LKg>MeCBdH?;mS4*2@Ag(>Aal?fCtt>t%_x`bPvwZd&p8_~!R_GEUdQs!Jk
    z2GhgLer0j|XCkY6WJ`bt{YR7KQZt;vui%R52jX9rOdwO~Y$pByKk(?)Q&=xtle+LQ
    zshA)y#1*OG%pXyoB3_Qd{q>uuS!ku#sBMhEBM_IryE%WA3rxtz?JE0lR%Mc>Ixt@Rm{>pGAe}-*E5p-F3OB
    z67E~3oK39RfR&0sc_lv=IejLRT
    zUPR4oSkC+xWA>V<%(BVX$Mg^uHb*)D66q4hMt%q9QE!<}qG)P78NqMUXEFp^R2#1(
    zn#UURr6F!3^PX>Z5Jej#-O_K`wbE-GCM;WgKp)u-4d_YOml%~iipLwKfunCHR
    zL^0}R3zZ@FX8K-y9I~T+=k8G9a4)}|nK7c5K7;2dAH!u{i`1`-to3@PpQ!M?<#v<1
    zl~qh7{1qkP6trH90gbl*5i_+;;ZO2?`j(YNi(r6S&NNZ0Z4C*{?yHbv2krOsCIN+t@zkHKK_UonC
    z4uRu1x4GCF!qt*X#PgbyC|+~aR`H^|9Om<>=Jf^}^BH&ryc&O5%vM+0qlr|oQJIM1
    zymJp_F}E0h<%N)^{FdVw*sOo8qK%PJv<&4eH5IVBi&D7Df=Q(eSD=NB=#>TF*N
    zf8XWjO(BA*Q_3hyyJxi+N;6y&n?z?bDe8RUU0*x<825wY+z>Z~ba6qz6tC%K;&#|tu9NsU@w$fXbZj=(_x%i
    zp38ou&G(nZ!`zq7yA+Lfh%oLjoM?Zm+K<@Z^f4;odvO^m=kvLAWh1{kmf=Ey&3V?f
    zLRz1&(D^|01M37s;!@;)>fYz~VokcpgoGzq72S3X+S9F8lAgw;drPz!1
    z;;nX(eW%gFHSvOyX3BKh>&1p2gc4>!!&&r_P9gwbz(+)PxQnPrX{R?A{Y1`k)R;aZ
    zj#|Ix?nzgvLS`SI50_F`$Z4)}Y64zdtwXc+RVb6E){&^6`#gc$x%r+E?nYOR`DJ*>O^(e6!(q$yL2{kvXX^c^
    zpTt*%_cXRoam0LKx^CQlPHeT_HSz2l=;ye_hWk>N;XIm5<*0!%$4A?Z}~KOKPUK@V+LrF3?UTaWKWXVIzXNY%->nCCacImBZe?nt1!3=H=L`=+X6ND`6#P}7ltI3I)%DRwA1L@Weae#3X{e&pd?QwC`BQRfoo1z^C`Jo(1
    zt)Ui+lsbu>@@1od0;1<^@
    z>!rx=9Us+guAX3&>o+TZN+X;BY$X~)8)9E(zoOqa&Wx1AOmc+yEj<-Az)nKY|H2K}
    z`~(gM%H6TQ&{ar#n}S7-?K+L3MY-y)%ej>&ajM}>Sg8v`k4J2BH)=Z$5vm!0iqL$c
    zjmTsV6N#}`#7gBFP}nH2hg-?LuaPG492kb9YtM`8gmARhGq-jR9v^cak7VvpH)8xLjP`jlq&Dg|
    z=WKMJ2u8W62QBh78v?uwq*3;xl`n
    zxE^KFJ+3^a4E+s;@mT4)bw4PAPq;&b1I-~T&`ixYdXalg><~7(kASDnnaqWn0oIQg
    zc7{mLu>&B1ItmBS1zBTY!LM)^72)>xLewdqc3)zy!WUtwG2iH`K2FF&uyELzqHc|l
    z@p|H%wsMCq3(@#@ew)48J5nxl=HpC|BaNy^WSYrnwgAn637%!JRQi?r#5oSMI0sA#
    T)E-5F#{)T;RLg`{kxltO4xJu-
    
    literal 0
    HcmV?d00001
    
    diff --git a/wasted/disco_test/single.npy b/wasted/disco_test/single.npy
    new file mode 100644
    index 0000000000000000000000000000000000000000..4561c4f04f1d75b08858dbbb53ac806bea70b170
    GIT binary patch
    literal 35968
    zcmbSy^?wxC_dc${EkxIMQs;Yq0`_Z$AIRcYd2YKit>6?m727=XuVvy=VU(y#|*rH@Dm(=h$(x#>{j!GdWv~
    zXPv<096xR5?3tscj+i!c?7071?=))itZ`-QvnGt1KCbK?VSp(Wm{@lTjRMmW)Bnd)
    z!=coTEKD4vZDtnXYg~!yxDc%ugA2L2krbw$G>JQw^1*DSkCV?sQ&OAAHJKmbbzxIz
    zEb~qp4Qnu?@y~E7(Ult%(Kv)Y!rTdwQNKGLlo~JnQ$yX@cFeD!Da92JDH0=y&@%Lv
    z9?70yS1WtCyilbQiCf1V3B6SY$bUq-6wz7?+`u%5j@^BaYeI8eU#A}($V~!G*}tVn
    zOgO18_s|848WH8H?Mf}L0OqS}B{j-jKV-m(u0xE6p68yT4o+eG_rXSe(DoMmD%VT5
    z3Vdo(%j3rQ;C1{=dY$kkTG$(dQbrK@PHPa(j+91@r^FgtBWEGaW`c5@X@8n>PFRyN
    zRZkCJbq`3(iu_v)=<2W^F)EVnoJ$+_vB*`re98mGuZ@dzO5t61jrZoAKyzPC^OG0c
    zTfNno^)N!&m1m(UpcgftO+;@M*5z9oe^);qnD1y#UnDlBw{_3pjgqbir#x|;&gs*933m)uFCGPF
    zN9uva-s_P+ltP1xo(d+=i}h_#N4kc1?5sS*JXsKIci=(y6FJ|uIC@6CL@pxtL`=-9
    zI**uEdZQMV;YrSG{HJe-bc}q436forLl(D_?RVfRw@h4!3GQ&whFyPJrb~t7Ugjiw
    zOl_@yF_uL;!UrWt%0wzhc}>+}Vj?E+ACXEHF|R@;^v~|0E+X~0`#pFOJ>njP&f{W3
    zG??SYT|#s{H`Xo5!?>a9=|E5)FRm*c9A?7Dh~m)Vls}6;gkO{ZM8xDkn+^Zw8I8RJrr#a-v7Plx>V^
    z-X&W^?-Wg7{-S>)r?JzqEsjYPS^S@;mUuie)Z=n(jWjlMAeJ>n>(YO~>-uVHgkB-^
    zGT`j;kYk8^l`k$)_#9IBBw*TtvoO8Paj0;>`k
    z6guoOxRC91xV(BSPFGu`OiAyg%y%#18bmDm|I!9?V->mCqtI@)t~R>J$_ebBk;FZx
    zVFe$;?1U>#R=UQ?gto#_R*&W|
    zp6e*`SgXx`r5YM29LUt;Zlx?_$PwEtI8@-Vp7$%wH!ScQfDIMV2x{t~0;e1$)9WD*#o+(M(E@n*;
    z$F`LZ@7c>tb(x%B%A%wgp5<2ROVw(Hl&7^;o@nmRhzzE?7~?{_L3Z&7ZZK0m^m7qO
    zg@}Bn5|GFYY^^Od^`G!7S8uL+eth~7{#!U0S`ayo{0n|{ZYBp8Jw`_}U33@sf@wmG
    zGKSL_yN(;CFSD*jBl|wMPKL+ZGQ()-iAW^PNTWfjF$RB4nb~XBD#*EL4gcmq*Jv|P
    zhm6&)629W|P)~%K!5R@jzKs6vQjq_V
    zSa^>?$>;F@!YhkRktAcA+QeOgz0nSbF?K53MN>?RxDq7?n@#_&oGNwEi!Hs#g|55K
    zTlj`(27V{7-gTNwNdKW&&-xT?8}%D=Qr*N3`V)P_&+AuM72aSD8xLW?=%VI17jkJ#
    zD5Y=N&avuXWqo80-8~v|jpf*=J$+tqw(ep^DPQQR+%e}tZd1@(cqsgD(O|9uJV6(S
    ztK)AY--ohOCb{FqVb+vL73HUhO-eCz;tbtM|DYb?u90S9Rm3u4rGT*1~`Dr;5#OlUr)VueiFd
    zB!6wmhtI{w?%Py2-&jCqyZ?pTqPIZ@$a#~?BE3OqwzH~m7FmNoPRNlZ{4R|PUw!>S
    zxsZmok%kq&jT~?%b16zM&3W4|xt*QM^S*$dNjx&v3P-pHc{PG9#G^N}cd8)FQ4%
    zNiDe;y7a!rb+C+V=uVT%MJKo)P!GlLbdq|<{1AFdrYmdncGADm>y6*(Sodz46@P`v
    zp~B$9=t84Hv?sR$u1u{|7;mg+=cNv%ySv`F(>O8GgLY6hwqj|Ps6rJ~2$u?rkpV4c
    z>LbI;QlHUbhI7P#1a!fz6C~WgQHGI7@U*;i>wZ(Nryp6cq;jdy+D1U?}R&3F6yN?!MY80
    zHlUHnUKN_=9SiUD-(hdk!`MdbXy$O3H+uUk!d6`O6ujtd$sEbUc$7uVR?=IT?TLgl
    zkX1^qEu|FF2jQB+qiJF6XylGj73R_z)}!bj;7D4J-TvsGw$T&_a|*^S)GM)%L1UMO
    zSI4p<%cK1o-XNP4w1_U$$EJ11Uf~1b@zennE*w_^^ytX)z`01e_(fHbl8~V+-#d^Q
    zqHUF?7KdCF@$u-j(Acy=AzNW>z;PpM%rEqtY|3q6DswB;H1d1Yg}-!vh^#bPhuSGi
    zpvn9)^>}!A6dITVPx-8uxQc}^`&FMO{wSsyGmUGYeRKdgCH`$zp>9~+EL@OyE-z*
    zxuIw;J;?Pf)PcQM)S4N?ywJ-ts#cHlYd&&$(c{#s)GNdhaYajo=|$1V#qeGKAbm}!
    zYO~!~?cgJPBfUMmllhun4OD~)WPfgKYDf3)?i%5dSWnl=)JFK>=4KVB)<39lY$p7Kt$fj(Sn^g_Wps(s2Htv+|q
    zgxMP~8z}f{juK6_ya~h0pj~IVT)(@XYvd
    zqoT~nz@qY6UHS?eOEnrc8?8LFVbj5*@0ZUMVum-CgUi{>(KYl0%2>?~RJwLgsO}k9&wdh;6CV
    z18LG2U{WuzNs)6xQepX&k*YnKBJ?W!kIEvQL@Tvn)Wiuy3)UVH-6fF{>4kf6RQ$$zv6PC|jx69Lp&6I05rv!B
    zBz|v{;i_xX)W4!t!^;f=8--+olg^4FDVHm4%1z*Q84DumMrTP;%SWQ5QW)p%&eo(a
    zN9U^(+0pFTFm1k;>MFx#|5Mj8f4J7e9D2VL>r6v`p@M8FY3MBHPk&ahs_-TQBj029
    zrS^NOD9h9DGykybzg@)02&7
    z$h~k)>Ua1H=MMQqTeAb9`uN-MW8)WpWsZ(iD|E4Q!#i9bsY6COvkG(
    z=mNHc2ZJ}-aH)wafojBlbM5fNI(KRVq_FZ5%pk@K6*bHd3n$~cXwn1v1c8iS3Ub`e
    zg2VnSGL!11*Ne1cCXvaMi7Ny<^f#eTi5o(M=PY|gogVqq)t#y2&z44pdKepwE=(mQ
    zS-l)Ml;1x1+B%%7rYr6eqiI2XijJTrg?`{C`UfJjbdNSF=nTz8PC0b6iuR}Pvb>93t^SyB
    zLRlfV5fiz_j4jgDeK$Oobdw$A_;5e=7MZGUU}g&)^3G#4=s|C^w&1*a6Z&l4HT4=f
    ziE*pTqTO9)?v&mZ8yh_nIfC}I*H&H>ls8}1*BIS|82u@n$(}PZt6tJ`!Yv4oeuuhj
    z>6N!Byofzay%SG~eIlpmhD>Yu3NqV%kDa8h-5n$RY?zqu!iyq(?N{yF@s-RDysdu)
    z+EiPkRwnxLIdlv+BzPTs7jn~~(6`8XxL5h6J)^H7jfwHVW;o$E(yI3*F{zK}HMc~I
    z&=!%3{*JWAPlX4Icd^M)D#S-)nV%hrrkd`pWf|!*MJ_r_ne`LOU3?3hPp-W$K3IC)w&b6
    zK@}l8Y^S!vi+U)ujP^PE;G1kwWRqT%wj19E&1?gB&{jod%aCq-=pZ~z_!ycSMQ`xj
    z(|TZI%Mi((&#-I!fB@`7GQnp{7I%v8hVE54ZQf$pnDPwb06)6qD5gYasDr1>Dj
    zHHRn{X=IF+KiJpdf_xFUBIP47Y#(m1eNdE$
    zgmt&@gg~)*^h?37)n^ts9t3%Em3CFwMlJ-Mkh6Q4Jvn@yqqed{dqp(Dk0Wu~2C|FN
    z34Ejd_%L*u_+D$It~S28FWNm?AGAGX$Y1zuWRoKxBqOyg7*QH{Dz%XY&?(?Ac?;h*
    z)n)1o)4XGuh2oGf@8}u0ik?I#1{?D$z$k%;Y)7x>>1tRn4>~A}i;HZ{c)@;LYD1n>n>*{mjlvePG5MTGL6x8T#(XGD**BOa`F*4Ttkq%E0x%V!ImOos^^Q>FQL_~m4mC``EY%BFaI=pYDX1P
    zB?bg*=u^d|{B`0G{F_`+(8zcM)*~NmKe4yq8Vod@FMJ9NhZW^#dKJuOqm+~K3FPgr
    zY_gkBpnt|X!M0=_anQ~Q*b}rFS-}y*)099L9^Aseg5&i$b|Lv0RpK0)eAIgd?M}+f
    zBeH48X?jGbrvAY6a=wF=)pNuheIq!C&H{VYuUL0yH+nig+t0{vz@^CIke^_%-)T%}
    z06z$GnS}w5bU}>5ElP$`6v&Xg>{I|k(|7D=+J@W2EabC7S80S?s&{ZqBoEpc>Vn}F
    z2NPbIBa6w~pq;QJcnGw}cVMOB3*az~AlLKTg;wZuwVDD$2iPTKydonl^?!LcItM&6
    zEn(-$74S(U@9JZnB9{|=`uMW?)Pt1Bc=fdi%(&JO_6H8E3U(V#gdn>vb67boyh0f2
    zh%%V0#up+>u%UPzaA8{q={Q~0kjXuGnwH%lM^VK%#!}9ZD5Bw2`qK|Y;LI$3U98DbXI>)=HNcMGT7%lf0!|M0^4lE&5H8H`4)_&
    z3$-DROl&-Xs5xp4dY3RZ#{&C!+N`yir_1|nJB()Df5@xC_rzB6CzSDSPCJPsaEa0Z
    z?Tt>@vxjMInn4&!Dg1$%N&aCQq8MSXa!5GC9I+ooO!`;vHFXePn$%sTEpf_4DIOc+
    z_&_}(i`Bb!#@a8?jaa3hV=f}~#f3%N`L_pim^9E|J;n6(CK-$Ax!5^*rsYR-HF~~N
    zpo_$FwjwYyb|vT}&*xD(Su0VV+xKI2Y|qeqWI}2QHD3D_JEx7RbXow$g&@_~Oj8V{Qu2RS&nQdi>x>>5N!*3An90H*qKftty_M<7*TqICkZwuY^yRQA+hR9G
    zG*mpc=1B-{Qn}zojV2xXZo3XQ{X>klu9Y>I6
    zB#P&NDa6m99qu4HfIs0cbQVEZ?8LVM3cMef+BCsPliU0IdsKX(m
    z6-iY8qRZ)~njeG1mDi4`(irtBal-an@U!!)V8BtzBWftOOQD4nI9DIx#o&mRUrS2u
    zbBSufULp%WVrP+)ureUw|4j4!hm>4oj{KWvz8vCzvplqSS8gO{$+BLIz6!S2o){P5
    z9T=zumSPs;ZbD^Tn7+AShN6-!ni>>0Le26@8QK^OSFu(vY^-lGkL
    z?aGDPEq1iPUK{7N*nMk^TrilH=$86GDWl|=_7gHsE+xA-IvWYXYU2zx%W2m@r~_8b
    zsH^W$9Td0sN3^mr7|XFa9s}9WyOUPBQw!^Y7qCJpCLWZ47E)0$UBYi!vd5Dc3VzDYH{5|xy
    z?J#*$8<93stz|oFd`~##l!7S+$#miN+tI`z+;>jc^5Ydf;q$pU-6)%`Vo5;9jw5
    z#H4oTaBvt#g-1$H%L(oGKt<#!))~j~rebelT*Tv;u7vbb;U!!R41O?j49v5h3S7)C
    z5S++cX)I_5*AR9l2Axi3Fs&%F{*TOo*>qPS2Y!L|^4f?;0|I?4j?q^qm(?tBv;H)t
    zyAp;5b=y(iSy{<|J&9D^YuwE49_WL_;u9Pu?JjvrcR-)9O&4$u8RgLUe
    z)|37PyMcvXygSwV@3I*FHj4|49#H_)x*;7%o5x+4(6hNNS`xFH{9ex^T!bcR)gCvcin3S#gi@<$^9
    zAL2+CzIyTWn`$||pV%d#Ho02;%X<^)fcNr`1jYO&YY&@^u7~5!QTaZ734O&8M7oxJ
    zS+D2%Cn|G7AJoNqp|&)xjpZG=KBp;OO^7;Qp&9iiCzKO^6aJMfne@!#A9?Gy_
    zCu~xrC!S4vy>FCj=y~-pQ%Svs?^1X04PvLTAEdp+vz+NvOHV^RRr(s-PyVZKcN8ig
    z>`SEdNG*Cbb^`k&cn`gM@U{I5cm%D>dZR+CnXs;9BK`|u#>X40q#?q;RIE-(xLP4J
    zOuokaK(3Gr=@$B7w2MCfrzAc$;c#Gd)A9V
    zK&tbyk&#f{SrjmA_teg&hHzE%B5b164m}ASL&m5h<=>>yxx3*Zcm)}ZJ))G|erG4T
    zu5C0;Ql0SMyq)nx`a$j_h|5QT+kajxGMagsgC^>2^F=lu^g~*+n1sr+lx0Xyypg;+
    zFfno^zPtU5`jL1^*ChqUkw=m%;R1w5{-IZcNmvHoB3L!?k#mtU+)3HL^7^2FY^9pcBw`%FSqHey-}H=g?ORl3<`rwXf_tX&OV)
    zwn*|ZUlv(kyc>9KNEB+z2MM8vVt%9+aR6Torzani-vpO68%?Kyrim^hOKOF4q^$Nt
    zI(shEZOF8+E>KONHM5HT5znFCkz1(@BtYyyU9Hx!3)Mz+ccDj6v7ZJ1GiGTAZJj}h
    zK9wH1_mWU6pn;mvyud3do;^=aRtw=MsWlia#70W_&PFi;Kud6sJ{_!|@Ph1OvB+Qf
    zx!|moQEA1JIM?A1OdqCJWd
    z@CkcPiC6j%8$kKcFfi13fmAUsRl4in)8j=)QBeI`93}2x03?GuVPmqR+Gg`W=k+vJ
    z%*QFJx#y`+i{1sFd$U|0wSIveWj*-Wq;`SoXba0L+bNnB(>#BxsX!HvAs(V(WEb@+
    zFUR2^J_={FIaF_Imj73L8QEGbq!d#VVgfTwZ=jE(jOhK4DrV=$SV&(3%HdCi^Bd2A
    z-Ng-LhuwB{JpRS;t7HrO&DN#HqQ|tO-jU(A*ahSOOrU3@ClEhqsP{uY!(nXfv2
    zhfO*`EteA-H=(LoPe^~lYw|{U4_a9|9Gq*q?b+u09-oR_5;FI^4ouaSnOiH%a*x?7
    z!K6+7v_TC}8)a?@e^)mLlE|>gnTnBS=mR)MJT1}WQ**i1vlJ6D#Pbw>a;`Le6BPLw
    zWUwlZxt3YTSU;cBS}ZXAE|`7IJwEfkf)Z(x%|%!PXH2(5r|>cFp<0YIEGP_)b~r&Z
    zsRVBahofJ~JhCG3u@R9UBUQxX#7V>}=RmNb>=vmixB|ENt>O(NIWSYWqet>GgBQg?
    zo(g&gw4suRq!qjsH~2MevA8fHkhd>C8zeN^Xa7@d$WQfGOdgU{2>&3un?70og9C^~
    zcz)h7?6z~0DYpv*I*&jJL^ZWnXgD*9hA#B$HvLi3bvH>cJG3d9m!=ZWk2PO4ahe=
    zC(zdTV6nN9MgAxgLNA-c?sRlOrkL7+rp8&PMVP*=JJJX23e>eTa;bls?P<0fZECF#
    zSKB%g7RoH;iHXP-UlsVu-;k&!-wDq5jYJ0Hx0N2+`hpJP_ofWJqGAx*a{D4)G7AwM@n#n`OZ%OHVH~;_St-v4X0^1c-Mt<;@+qy0A
    zF@CJSx%DSy53GX4%K1V(8X#BLUm?4Ks8{&EKS`{_OK>)E&RSjED)#34+Y^J|iK~2#
    zv{8ZPULu&Xr;5}|$rN8|n|A$Xs|fB}62%1R8CE&)E5DRK0EY)k`2k>gKo@(<-^3H%
    zk?=V7O$cKZY~AceaFBDt&gw)pV=VETP(wKvJSX?r`?F^ol8;^%?g&e?HNkblDl?zd
    z&iBezSsK0fvNXWd!-vXcN~BQTH-fAanP2~whD;hlNnWAP@QE*!+B1q@6CJ>{i#%6H%L)|V$w_s_uAVPbaJ
    z5(P8VH+;v0dBQECKFx~;_>Sk|zJM;6DL-xdU2g?(5phlVJ6KV@lF}2M8hi*(1wMgx
    z$^va0zsmZTRMFg+eD5ceC_3J2N@)UHINB1SzyT-7ui;&<6aVtHEK`pYYs*+DMW!&i|9+3TSNWhu;JydWPPHAk%imWZ2&xrOg8~u@}Zpy;m
    zBn~uwLNoEYNzI+_QIAkB$XS=tF<^|)6FCqZfVOp95K9u8d&e__Q?9~$&K&1DNJwV#
    z0ay%Lfwo`?tnb|C$yE9vosjGCDZ*a3mrcSF)e=V=^fuN{yF+e4y(@`x!XNisJ0^1yiDmSyjW`LD
    z=n(RZIF0p0nt5+w7n0ub3muMqe<$C`SxTNSJZ66q`_05mR$`H(UC}?4O3ZnxW>af}XYqK&PdCp`G9NAY72d?=V#CP$
    z_)XI^rk!#CeG2x#{!$E?Ck%&gkW5Q=1Oo+WNp&Lq6>)KUKqj;8EFbI%Se8DB|MBYM~GPVSaI2A*wMhHb;GVm&ZG
    z+5t1*G>63!^zOwiehRCgSo}WwYH5aTFnDi$3VY$#aKtuwZ@y!j-W4>{uAB1`UOIU#
    z8CJ4xmTvE@v}1W$?pKpq2z%vo3B5qu;49}N{;T&mH4kPAwUuO|1nY!NftyS*XjSkh
    zvd3;##?z~abJ{cKXDJ6SZdwT(^A-R$aKt$Uw7`ma&aqUS%5M*DWYiPVNA#4q6wY5{sss!it;5v;D0BA+|f!>jy$v1?wA
    zO0fS0omwgVAev_R4iu4ISOb*99y|L{Rro=EfQ*5?{GGLC_+qUP7QE2TSBy#0_DzIu-i{o0FWhMHz(jrBB?
    zGMz8s^WmV_^Y&}loA@%#LOf#1SBvp40SfJ5KZmtPs@Zzw;P8-xG3};eratnoriz3Li8`N|^tUz+Gmy9TEB>bT?!FJmrs#U;LNC}a
    zi!@(Bs&WVscOnDOdHTYFC!ROn`@rTfAx?7TzBXz^tozH~b<%AXw6T&e{%7~`&+VCj
    zkUsGXa`WN4#Eo#P&xai>nD01b@YD{KqAf|Y#jB(FbT
    zzU@Eh*_4+fB@^TQz%j{J)zy$3s%$ZP<&(Cy*n6dS;u&qDo~f5Z%gL3n$>mZ+Z@iEx^*|;os}dUKR&ElI2Ah&WGh{na8%tNK5fif80@bq+9$kE
    ze6wFW;iw>Hk)6qV$YQdFICJkR{5!Rr2rLW9ANVt%du(OprelwDm!Oha)Lg)Lg+Jo
    zLih$&)E|H~Cs?G|xij$y(gj(kAZWJjS9GxPU9g_LSd7C;&{ud1BEk7F*odg>nB*N+
    zD8sjj4UtyrYP3eeT*qONC8~=KK3=R4yh_(3>ytLmHL#s%;VYy|?Yi*AUl~7YYmNxu
    z6qev?LC4S`Ac6pqjAX|zCsDd7s3Ek+7MQ0?qt%&Zr`(;lVBhor^sEY(L@Ekrv{9b&q8Ydf%2q4Ti6FEbzO@Ax%SIqzr(~+(pf?=Q|JBrh7y7
    zx8fEsC@DLz7k2U=<4NTj{x@yl}~XP57cu2__?-v{4Dm;0-h$
    zf#$?u(5PbFf%veB=GDub^fN@}0
    z{@%j`Cg0fH986TAM5p+(&X;A=|C>z~=S+xMY{YY-8`fWWr}su#I?eeNar5q^_bP^kHRnaV&9Db`q5IfGD=s16D_QxsK%E9saRpDCHeKi6{t8*m-?V
    z4^LZchb@Fx4UCeq9Cj^;+Ia49(IP6
    z-dK;oL31{G8@x&sKo+Q~(6L3<0azWZ^PYxrB94Qg!&TO;m8(o={A=sS5c+%E>Dd|Ma%RPz*lS6PR9F!!rB
    zmZm*Ztv85Me%%t01xvB`87&t*C*MPB+U`-^^mG1&N>_E5|D$si$xu1GEAi*DE^ySw
    zsq!w~nRrt8X!+i|9vp{vbHCH-!aWEZNZa`j#@VhEOhpXGATSU2Aiez;taBWfQ8aNg
    zPm5>$hjNkS
    zSXmQnt?s}VN?#+D%GAp$R$ea;UdS6z6h*)%%W{7eag)EYWe3t;#Y6*|z$e>x@k;_r
    z!8Bwx@)N(sV+H~7J5mT;k!qW#CoCXW@oCnOw3S##_9I(bz5H76pUn+57`r!;(9V1Y
    z8A9}Ga62?ksb%vA4}$%8ifbJ91se>8IO>=$$r%ERoyxDnAoyM678--MslO7Q#n$C!
    z)S3rZ=zW~&!6|%sN1_DcJ$O#ANhHPj-hWUSYF&-JW*_@+OW*0u@llct8-oJJQ7ktwI(~^G22RCJpx@L!iTA)^Gq!h#_8wbb@KxTHyA7{JX!1y{
    zsd>J=iSj;qg7J{AhBSlUDYeQp@tNHFh@x+R7W^NjpS<2*RB%YWOg{0f#s3oaNIkTB
    z;y1WB_|Z}iiCBB9y*B;~zf=fXz*B_Pe_Cx%dchJl1JE}hRqFh;&hEiGfAZ^
    z#ydD`sjV1BYq;}`;}7zqnu$=RL~&;s)3m~?1SaJ6G3|gRa!a{uNK0yt*2eo^@ELJR
    zZ0)U~{SUkrO$AkLx|4#-kt8@veB2s!@$|USbOg3fr7(IWi8H{CYH~CW$)+sDxdT=Q@eXc^Zz-w;cjUe3X!MI
    zba_+mL)+QB+v4A*%uESukNzGPgUk)^Nxh_q=`(glYy%+ZV{fYu7uqz5;XnJ?uUEa?GDdBX)Bq{2e>%UCG(%{GZ$UtCz3-AO6MyfEc)P(zj>6c8z@bAD
    zuA$EbG|yqZ>*(Rp3+mdYp|W@^wx`XEKHFP6DH%OnWgI^l9xS+SeUoUjQ{q~I6`ltA
    z?O2$5)W6>0Mcc$}i9L>f^8I3dB(3pZ@F(m@5~q0X}py8j7LlXTx^nb14*;*Ko;jN
    zN9w{wLIQud#vITKLz-j;D>>>1uEkcCZsu>1w+Y2~G!Ww%XgZE{68q<->;~|WsQB;O
    zF3G2zl@tt&55DlVcd+27JVzKq9X4lU1H|jX2onin
    zn8X>P#naheUz#gs3f&S3_}sAo-fIl<<=CBI?Y7gl2cFsfo`Lqz7Yy0P__pC^O$!4$
    zKNtBiZyjux)Yv!Myf3y6x(FYHFv#%y?t%09AEfH|b^D;*uHX#eabTe-&1u+XCnq}x
    zC3Py8?`#Xw{f5v*+9m#Ad*T_np}KS;&W3s|tvwSgM{TNQs584@mi)x?Gj9ua4eqxd
    z7i~5NCJ3dT|JjZUQGZ0LD8$yyOmh-UJs1y4d1OB;G=KDr}I?;FIlrCl2!Mz0u&KAx;lV+gF
    zf>(ShH@6&dPV(paXQJ=K^HQ&R>0qtDhNWy5wL4!0juGxTdkKkXcHRlwWWM~~EyQ|r
    z9qDY+bxBDc*)+~F#{3?gAS@Ke1mZp0AP^V$vVF~kkg!U$tsRyeMvnMi<(8*lRGrrPvcegW=8UF>4IR`+S4R^bKMW+WpHp*`mqjD66QCXY7CL#xC}oEiT~A4Kw{uk?UhSIaW|`S3ySU4?WgG?S
    z7#+Cu)PQqB%7~O{e3)&`O)L6UyX2bY!c+dF1F5Hl7gU-5k)CCYO_}W)5RPi|Xx!n8
    zwq)(*E0MM6{>V{gyytjYlW^P8-bxGj9aYhDGu+fYS+9%nY{h_^dXAHkSJ>%&ebRQM
    z)rkC@63_IsO-wH;9`BS2Po@lwRAS0=?eaGewc+Q~B67Ryh5_lGsfc?#-Goc!9!578
    zzT_@ZMQ|$@qy9ye;|yjuy*8Q(Ye%Yv3d$Urcj#u3IfYA8bGcWc{@OLY3sJ%Sv(Kz|
    zkoJWpp}A$V{WshVim#y4+yWh6^uc*b?ykRQvjd~l3(@@4zr1$(ATz1VN$g@zg?F-(
    z*$PN+=I`XF
    z^i|qIeW>vux;AY^nRj+)nL%eH^^@B|J=U5b%iJM;WY9{UKo<$IR0VcX(Fm@EKF@f-
    zeGD)~7A`$)X5nb}>}cI03%P(v)|*GCi=}Gc==V~kqC@4Y=$`1{;(a00`nj|#^3K&H+9Mik3=DP6PT)AUOX0YXAFl*`I!wmF3+y%L
    zkWL2E;23wvjYLE`QT-+ISD>%_Z{$zLLhf>e9Lu<~p-QUX9d>1wUz)N_n6FPJF8le=
    zR=siK#-WTzeyAksaev}6!6R4Y@O+_fI8$pE-l7d6|FOEXWu;vM@!I>Ss?4AlhqByb
    z^@9MawWALAwk$s5XlNIum*)Rw!3ykjx<
    z3FC4>6*@2SNG1y}!Wtz#-7kzOuzFmC9vS^bY)dTFUx;nFSuK80NU{~ZQVGHZ4l;Tp
    zLa2wuSl26sNQptWXcxn~m=xwVvp2GqZN#L6`cl6FXC&?aadejPO`U%mhvKY7N-1@X
    z9Jib##l5)0kO9M83d4tV42MCnQlOHgk>i$=w3OoRI$*$X8#dg=*mM5R`@G2K&bd#1
    za$Voco~I}1bp>BqnRG4x5%4u|u@+i8q
    za}+s^TrafQun7Xu3aM7OUitotli|wJCq%3dk@llEOds?|^k}jNwU(YuMA<6%8yPm@
    zY)P_GSDzS~7wRte487X*hPzX|fgFUNhUSYC!OM7^(4XNZQh=-}m(oG{IM7vDz~lh+gXy-5!YA89+9Y(#YN20)UI!Zm
    zY~dnleO5i88@VdnNZ84|Q_kAg2FLL&oeS;N%GT(EwO;xe>ZI_eEl(Ik#tWmNk(3?X
    z0XJ{f;5Rc`nH8jd(m_X7@#I|-@fM*|)Nj%-`6F>OP^Mh67trT$Us<8;&6W$nELHGz
    z5
    zn5$mFE-E!Nwu~;HjhmS2^a}hmf5mxTJ?A{-I7Wlm4)D9saq2XA#Zc=dI;W${u+p+<
    zs*7}1GL=n-yJMf^St-krD6uN>t1Vy4*IJ6-IR?Y0@iDZyY%|eOJr`U*@O?zhtm<@Q=N!Aox;{9{|29<18WoDPuNJytmoTlso^>PGMc>u*xz;^+%{(02
    zLq%9$?)H=QMASD{?}0qCYvN7$T<8^BsVr(5uQVd_wO;foduJ7wt|~qGhW2a;;I7k2
    zfo!vrJ{s6V7uzDUWMFadZP{_sP2Hj9I{prIp*w|=fg92|%OCI@B}43JZ=#(oZ>BeB
    z(p)MqPCQaPYl#DHs1KxumOr&5Y=^Qc{)%2oyOsB$#`HS1wtakgZ79cn-cdhPASTHj
    z>?4uy6}Cx&*V7pdF%USX#$@e7?x?X86F
    za;0g5-cra6CnAf`uh6Bk_S$LtL;Ak(nAC$4#hEQn(yio$+9@hh?MGh9DkQh-2V;Ed
    z34J$_6CUARg#4ZahwxDS-PwlDtsHCP7$~(3FV*k6WXC#MQV3)!wVlp`QtU+Q(q?;5GZtdSX@#zodCvC6c8pju;x-
    z&Y{iW`AWCYXn!;MR}xnn*ju2mWpcnSYNBDpy2K6tL#Z6l-6*2A3D&j2#3u72
    z$2C1~wSw1T7Si{Cb@~@=B6dP~q0Dve4(`!f(!0%D5Lxo={7{;l3n
    zG8%I9nZ2i7wD)#Y3%}&L+1q5~Yc9PJK8(!sPd8D{zGUCBb6TD>TWLi{(<_}n1XF^R
    ztddZi*4_D*|G-k<6*Wn&L;S7`q`}aQtV_DlBdIgM2JCFz5nK{{$i`B$LRB0+919$^
    z!+H9avf=g&mg`fK
    zl#}+R*r#B$D_7l8`c|tNTIT*)?@9F5vg~Wk>zs>9+XU@+2H6aaw7Ih`()I9e!HQrd
    zo)MZxR4>l5L&!z$tiLUN%h$s-E4;~4f`Q@AIFD6OUU83-khWbZ1h0~t1m3aFQ9uKz
    zDbz@KOytDAMpoII(+h0t%WlF=B$rm^-xf@jA5jYPOx>;Ck<3X68Ae4%FQhx$PKm41
    z#+eV;$L<2X^ROh;t85{+3pem#_QBd^Ye+xs=`6tVKTKh$dDi+CwU{c-Z`x4%8FEY6
    z2z^~Rj;e}G3Ul_3nN#(1xZqr^HVu{qBj{5yT+ZkP&>M1#c{u1mE~$ISXUb3XKKe>%
    zlA)yD8D0afq8WCy+M8@8A1#OsL*co4^#oTes2x^vsbfk~&?|o+YZG4Njtf*9gj+j?
    zl6O#CPAxqz@8xRRAIeU8u)fluj`X(p&=T83wT&|`n5%pfUXgWe@wO9Ke`QUmADv&;
    zlkaGKZddZ@gSGMt5|(Q0$o5iWZnZQ*o2zUIhxGlS=F+ON1bU-Zq#O;UYbXxdQ$=Ozo2Wo&d
    z?K6XY=tMiFE9ptW=1xGVM?Qh>+SkiZgC6=A`;zDgRVcaTb<0O;u>LF7-{cNWv)R-*
    zuK)d`CUEUo`!MIh^khglJ@+08J8N>}M%mPgldv>Qo!Y3c^`FUL9$R
    zbYkWG$Ta5)TN3;guBvX9+wg0sfZ?>f#6KqHQoqnoLyJ;q`}@p8#4zQcbC@Nv?5_Mm
    z7|SM*kJV;mU83CnpLE>WQ%S^E+Fz(_=&;s?*12BvGheH+HTLsjW81^fpV&2GxcQ6K
    zPww)qq1)iYun9`JQlC_%-)&36nQ0@!JIy`9>(Rqv2m3$NEMb(sxxBV5f}R}wmHte0
    z6Dq{DY>aS+;GPow90gJS{@pvmOCT$tHd$yG3kFQ&U17!IfKv68csVA
    z7rQd_6rV07>Q9mVaEd&Yx?;aVpW4MZpTiM)7k!xXDOwP^N@lCagWu~LmC=sc&j0j{
    zwqL^o^&I+T*&$nZ-KrhXcaR71e}r+?pTtoDN9`%wrq9xHgdF0Kenv}l-P<)m9dG-^
    zwp4ng|6w?vK>LsMG~#pUD6&&|N-R?1$T)9vuy5s1a+TB^P?2JMjC9P_RPOCakuK}!
    z99FU)R%zcJ{9Eg0?kKp23;L
    zAHze5f3S{aoz}~otdAghr#zmUdr%bbGdrBrMe#KKzB}g
    zPP|3h0cNaUs7PsI+9Xzi(}?dtOno6%p~-suwYZJS5cez9oNf7>Z!p;`=KE
    za68q!P@ys_rismJyr*{wb(8PoWW-
    zYmQx^w-^+L_C!$oLR_UUvuA*S*gdoYgmye)MQ(`eRXh}A>
    zqhr0K#@fGPJu|L3M(sSUEhH;6G
    z75I3=dl1$p=_jNs($Ug)!E=nn_Tp-aeT6ytzTh&|B_^m#gOR$A>Z+#7Emd4Njb~UF
    z>f=BzFd;Ocd@N%kvhfyVSKe&92`mYZ;6GDiRGr8seiJ&TA=bCANRXyX1N7GwpYtPM#zs(s#oZz))d_+hctPKO_KKbL6#n5cdnq#M`=A
    z^-}w=1^M^HRN?|-p;M?mrm+16oNSsQ%~eOLZ-95`aAbh)BhR`9d5$WKEhgv}S^Wd`
    z%5lLPgMEiKQk&@{`w&knUnrf`_A|@jV)X=hfsn|Tfdo^wI*Cr=x7*L8Kjov80k#PQ
    z5ws}TQWYVB*{1f*=x6&yC2(0iW{T9Jwccnsah-|^K4BjP*AXkwxmFCEMqMWQTN)ee
    zn)tNgVrycu)>*454#IwsZu2Y2WBdlB2~)>#BY%?x#D%Zmr_hzb*cIm5$%VIzog+8NW^qSV4G*!0B7N0hP-X*cz45B=gtrXctYW<;$wqx*m
    zO-0@Cc``?I8MLnB8EL?uNT1X%%4yT=|~tRZ%F0?``?1xLg}-h)KByc_$EucMd4+iZ~XQ22{<
    z>TQg3?I;XE#&DANF7uUHnDItbtgVtg-s!N%{n>jL*iyRE>dTmnSfOeRT=I$O25Dx^
    zFZ0pFMd%EIy2HLRN$;T=uA}}<-hEt-`!?6VWVX8+zW}}knvhgaG>F9QgsEa?U=nk3%lWbMqij5~YRONycfpdwE#NxCT?5hP3Vg3vKe;*58?1#M;(BE6Z*g*GGIS*vmEi2aWYNK_UWo;)C9v;B@a0%X*{`S%}Ri__#dRE9;cLi0E|t?e7i>nirR4;M6Lax6
    z{~mb;JXpweMWU~P72I{C*q@SKvv_#HOVbo%6A(fU@7&Rg5t9%>H1
    z0+R|G^IuBau|II94KCmz1eUAuNyT-!QGpz+(p!f;3A|?BSNsg6VXI1G;eOyHUz{+J
    z>?|KNw5~3o(6>L}F#D7S
    z(i)_@bv>7n@`E^wod9-+Vxj9`A@b6+jhl|x*n7n9{)wVDFoUawW0r-Gru_)j0@|^P
    zupM6S`{HLzH%)b|H8Ll;QDTJk1vmoTmv|q=O@$i=16P%}v}N2>%fCoFUrkrJPm%lj
    zx+6f~xY=wGOZ@&>fv;c=f0VBuSdKQfyk-qj)%JF<8PUOHVgVo^OI)W4mqT|pP3C9q
    zD1>`k$HiIj`eOA!FYu<&)%PGU5Zb2Nz#S%sp=OKXx_PIi{NSwx4@c+V?F01#Ev(s=
    zd2)vo3$w!fr=<+(A@t4sRczC+lVzW$N&y3}VVx`g5SS`faCz2Zdtam&9%+ssuA$!-7K1YSso@?p1__}N8GoYN%rAW1eG#P{
    z(V#guqd~p%!e$^-j`Vi|>IW9HH^5rh0oOEi2%64Rg*zwlRvl6eZon>fm1#)mFZN}{
    zu#E9YBVi2VWA-Q?2|In&@C0nF&;dwAmX>bw
    zG{GB`U*i8TFHLnZZJ0U2+KiJXrqLqdy>AZo(A){!$F$pal3A*B0#)viSS%csudq|t
    z2xKLC4PO(Wi!6bLKCCDQOlMs3LJUb89~c$Cj@bf?Oj!nOL+Y8YH+jT#5+4Hl*L_4P
    znajq$N?}?781|HZwc)Ctj1$GQnAi-=e*kF|_t{!od6zL9eq>VENnjC@19XSZn!uu-
    zfx+gjNV=&X)Pw5)oi@GX_s35&%?FqJBXPGkCwUYM%T(do#D-QHP4fOGP)wpPF3pO6
    z@IOddV(Ms`M}83*V7c)&ZHqlbv&3cF?qRa800NR@l8^W?^I~|j?=zFeO!ohSMKLwZ
    z@h#HZQXlA(xQXbU_SU!B+mY)?-s5jupO~@&9mL1DB8q{n++E=f{~>WC
    z?=fvMjTig-vr2Z9ZUqtZkN&&B9kd%_%7n_DrT5Sw=8pwO*qidqguZ+OW*EC0{LYJb
    zLts)Cf^0ytea#7H!UMJ`I?ix7%@N-U|1iH+h<2hAIol#^w!p12Ln&Y2=E7_LJTD1Z
    zQdfFMh>`yKz%=Am$#UszU?big>XOmEsA}>>=(KN&f1~vd{6e^gjX*Pj$AS;+>B;lv
    zGG6qX`!yt5J7b+74qJpyF})|}_-2B*SmK%4_|7g1I5*`PV`Y0HAGsPOKln$X
    zyL{DvKZRomEjZF!?fr#_!E3{9g@aJEDaKqAyblJUdj_NN9=i}I+EYz@r=IbjOxObr
    z6{q?L*H@28{wwW)yDIcGttrsX{{mVA-ANp5DP%yijlalNvGgk3o9s3xW`0jR5MQCE
    zk_Hg4uLXx&;#doDP#ThUG|&@uS^5N?5x>Cg(i40wkyqZ1EG>UD)$x@8hq-0UYAnTk
    zLdn^P@&G#=I21P-9h$i)^FP9_{2KU0i2)x0Uy@ItBc-)sEnh_7Y#ijBfb<6!NkJsi
    z)7cb_OwQa$Jm5MaUc5S#i+pdP(iR{g?pn!9H8c4qaE`TkFFxn~o
    zEAOf|Fzu2r3gx^R=2j4mWQk_HE0U4%j)kne)Q10?-333|{t$1e)c4T%Oz%P8E3TD#
    zhFffYhNXi~&^YEr+*!7Z^}MflOg~Y#lFSJ7FWD)yL`Y~dZwES3
    zeNgVMV&5C|$y?>8^Ih*&#PPX>+&lev6JJ}Iyry+HqVi+^r;|Gf`dAC%n
    zbw;#?5&St}?2FzL?O!jLfbQaY^L^|$rjFkM_495n`m|xH{Z_*49j}3IfqZaepiqAa
    zMqsnS5#$1B-u4v0of*meu9SLCily)x?=@(ivOJ@&|6r;hZ$OG6#D5kUk9=YKa3-{q
    z99{B4%m$`ddV+~$qSs37L|hx|2O4wvxPyCaIS8E%)D=qjooy0wM
    zMHwWk0B{6~K`=1_mie$!2fpZOmT^xysAme*rB|k@SO?!W{7PK5P_ec)_nzyH^k);?
    zEunATJHlGNs?fX;4dih(anygJ1VX0;xTJaFKF*C@0<$t6l+FsIfQva8Iq3h@*b^N`
    z67kmPZN57;O6ZbhDXHn{#k>|rSSthb%{Va7JhW(Iio-I^|556f=1Toyxsv{u<#I+f
    z{vhgwB1&EL(R7dRccs1%fm8*%0e*HJ+6>yLgySE(k8@G{H|rZ|a$#d`G*Ekcxn~j#
    zgCVgi8yp!ncJIJoe`wC|ID^Z3$0Pf{oJBDs;
    z#7qKFVIJ4Z_cO2%d79Qos^Rna23jV;U&SU!uJ5bN1u{*4AX~O>R!+;0!OhTh;Rc%G
    zotH4d6r(11U+8&k8C1hu$ewpC1-f`^7<7~tSO?;nrGAnLM1-O6&%hOCp1DAYkajSy
    z$x84ATq!ROTt`1zvOOX20~pD_P>LXz=P5Xgsn}ZND=|0rhh0CW-{-pd_VM}1O#~B4
    z%wxE2;JjuxSRd~dE+Q-WN5DwybHSaGodkMIvBSVq#fBG&lMI!*$$!AtNHy9=)+fI?NA}9}$B>-vc|)H`pH`3-T>s5#+-DNCy7IHBvg8{zV9soJB0+
    z1nU}fPik1$#6_B}nH!SjhC*pHG80|7dlYXyt?)=S9cI)hz(&wbs%T|zSCe62Q~=Qd
    zO>m7y+!d#@av*4T3p{o4M$l#`yPniWu4iAm
    z;kj&ga0<6P(9iVNJa5|vb2FfwsrR-<*85)04H{KvDwhx!o$+;N1(RZmHO)m%FmY%c
    zk`-Sfondc#_Xeyk!?9L)jvJoV0vO~g_NB0gGI9+U?JMpIo40LL!Ab@eHuM#nD%`uRVV
    z#BdEt3b?N(i>VOy21sV7f2g$CdIL%a&&OCyV~kVKGQV+KVk)6-mi}lv(<``1Ad1)1
    z5;J!|*E}7Vuj$1Sy|qfMeQc5WD^oXPzty;X)|bFNtCs9A;p`QEc5#=|@!Vfrk?)On
    zlHqFY4Se!By-9#OFw3`_5iL7RG;hne6W7s?nHX*g^dyka{>)8avw@dxF=?RpPF#Jg
    zX2wtcQAS6&&T>;Y4R5i`W^${<8lL5qLZvUpUj$q$ejc-Ko1Q)?X+>r{(;ot%OlZWW
    zDzTeVf6ExZ;};9Tqd=~o;wW$*yUMd7DaY&a@A6J%!u~2iZ{J+^GS2|0x;xiXV9Ft~
    zEMwyH6CU|0(=&u^%qib^)8*t3;AeKJsdmaOc4y*Swk_}lZp&H#X6uVo0s4jA%G}D_
    zU^>9K8aaIr0jO}UcM#%9n-@34@{9EevdLQ2RL^hokM*zjzT(z;JFp9U{nK*`E?~RN
    z-GB%1o$ZbMmbi6r8B@rTmPO3Lj23b4Jhq}*rroXr?Zg{?gs`77Y7yp6FNO2LE?%$!o#R`+0Y18^;~ksImGNPq%)=0)ZQ-onPCtnba)X0GTM
    zb0`V0yae)1m4Trtt(YUgF?g++fK%NjxF~(H_`p2TpIqF#@q*%}CC`&0y}e|Ue+7R5
    z7y}&N+&lHW<*xe}JHXun>76{U#KaV*?gv`?yy=YfM`UHvNwcFQ$%iv(4KAC8rmgTj
    zH~(vS3ip;XlLy#%YN~u1-NK|RSBO34uB6!)Vv1~M4EFjWaV98g4Qy+596zYO6`Z9Z
    zIZEk;HD-Px#o$QW8J)+x!1_RRutvgAM}@+=qVYvSZ%@3VF;d0T9*$1?1z#w$fl^sg
    zddbZl+cF1{@3CuJqlCI>tUAl@QlBHy^nE0?BBHDnrDPTe)z#+WXYIW<$}Zrx&~oM^
    znM2OoRgBlhjw6>!pTu1z6145$Ec+CHKGBvC)M~(m!VpnE*j>>|CB2iq7V?Yjq)-iO
    zNUTS{N4_Iowd|=R7(4R4?4J^<*5e+q_poYd^N@Pd8t-DIp8YKPn(v8=1e(>8I;Jks
    z%WP|4)Ns6|CL{Do$6Cj9VhT1*xEb1y27`wwNgbwc4-#r&t8(&%cpooQyV>%+2!TsE
    z_@B%Yt(&brKa?A8TW$L+{v<6Yr*SLsdBOJHsl@b9l-dGy(HU9aaqSFs#1-53nu)$o
    z`lxI@gLx;d;l@FL$sC$Z;mxy8Zf~4A{>{uxMTUMV8!sD5V;ze6+VwUM0
    zb;2^7IxS~gU2=ue%uj25g@W|j=zWc_HzR-6+t_QOAbpV;ruPZ;4?B{dIK+(KgGZDc
    zpr$o4{eW($vAlMrMDk*%xd&+w1q!arGTUPQRMKG0mC%_?kv=D}q|3G?ZcmDiu2ZVo
    zCDKA9FfeTvv-N4gyTJk}R=tRgx2@wEVTK|}xlmq8kCK*oj);TBi3A>esPqDR7%Hwc
    z)C-S^?xA$0>LJaH4aHP-9QhZr6y6uAl%2%r5?bg%9^P`-sDm|ock-C77;2}-L;~#z
    zE_W@G?*!LKeJF|Ti;py?g&Alu@v!_4_$W1>+o}IS?Tapj)2UzaiwRj|kWZ4b
    zp>f1Qq64W(&6JCl7;H0D2ie0#Nh@qqNgeBs4AE7*CEYmH$-YOBP=7pD8Er@SpXvF;
    zF?1vPK*&M%_!_`8k!BwO%ts}UC=}X*YIlQ)@Nkl{ZF}N}Cy$~(cW1dKv;-=8R
    zJ^g7PnnZo|_Ruc{?HZ#^QI4y%kf!=zswLW8S!bCO8tktv9SzlIS^|xzWpty^&9r08
    zCGRPFTic^Ro-)R^CiqQ&q&LYu)ki?b(5w`z{cnAtGE?mYjWD{!W=0L0gn;@1v7DA2
    z=h*%J*UCX?sFp%}PO78svGpWQ606lU*j3Q1JTY7u7F#9KhTLu%M>GhIHJH6;<x
    zi75(8?6nW3KVl#~&wEAx1fIkG)Mxo;=$AtaDw}9+%BK?BHH*4d-lT0$-^nf{qmwSm
    zAGj2<5}j}BA-7h}hH`}SjGsdEEK_zN%$FQv8yf9Cpi{$dZLJl9>!8B=W-?CM>s`tRNT2PwttZo3I18DSOl$^p
    zL3wPeVN7<9;bWB(Y?yxPKP^?H77**qGsKJP0oz=uAL=S;2i0`E)f|n
    z+2Emyy+S(aPzU>ls-%)hul2sg;_X3WTltec)qYX9sjQ<`;ghf|Amtb|B&!~89
    zIMqvfRNjs}j4Z;2@)v`PprOo6;XC^#d=7P7JAs!thal(G8Oji~syt4(fIiS(0lxqg
    zdBkklo54+zm(dj5P(;`t<$U^2ay5Az2nm<0y@M}XY!iaY7w8!94?o#907y37RQjT1
    zhGMQJf^0RhI21{oK)1^gcs@6ms!cQzx_Ecn=F=18+4?uKq0$sGdXmsFptjgt{|@>@
    zTvA=+2^`i2mcMKMD0l!(@Ruxm!jl@^wEIAtQ=s&3RNWbjWG9HL|WIeKn
    zQV@TcUM>F!P1MB9iMCt5Bsz|qsLwR^^5dM7@JHHyqGPZo>Bl!i$Kf$(v9ek{Vi3>Y
    z@m}g1`ZM&FNH>ikt|$E+Oc!5=j#?LSlE=u3ociKOs;PiC-VzJ$zfL8iD
    z8EIVYusu7EaHEyu)LFH1`zqoR&_|kK2B;W=(R|!GMrlqun>(pvr7`xeVkalB-_&wr
    zt@>BJQr+juwJBPR+8M~er=xR&RfOF{U%ZF3k3AWjXKYM+5t_}V|ESvNDNInT2(=R$
    zpkYKM`kG#{E>59_!~5{M!8RTfoTb+VJIIo4j>VwZG25k9^yau!IttrIN3b=33cO_3
    zO6sv##PydXR3~pX1#R7I3BU!yWb(4@ZQo&0Xc^_9FOg$yklI0h)i8l9?09k9s?05^~7lL6W2=!q0?6
    zEh$S1?obMqrF6bHz4)OWB3jVfY1-yuT7;-QM*I>tbyx7}Us2iM#O~iWRF}y2l
    z1C)mRjUCXt`{K2>Fsw_=*8qq)fJOQOaQ`}y9;i6=c5*CT)Am!K0IRC~uJ0+t;rc1L
    zTB&(#<^+9UdMfuPmPmgp+$Bwf6STVNlkq71jcoz)2RnxR$9~&}Q-dv+k;O!-@MQrH
    zJ$5HrS`i(Z{)2DVzvsUplk79>exj}RAAFj4E`0I5K?PwVd)xdf{+RX}>42OkZd*(E
    z{eTDl65Q>dq8>yV3omzq$al0sUxcel!~I8t@#IVPvRdBQhm3Lzx8z9^!?V1f&>Ve>
    zoPneI27iPePcAfV$DA^43yU3$oEMLtYF~@mxsC!aI*^)%YH@$$R%Z*ohIoqVY226+
    zKSzy`suE{~Q9Zq#kAk#hsdM
    zp446Z4ve7P>Oo>EzRmd&Z%+M0Ojj0I+M74oI_dw4{}^<@dg9q2rAOmWL)*e4)=NDZ
    zniiy~JxXn5k+ROHX=mv5kzCUPWl)R1k$;I}MqM_7*(SH9YbxuDCSeDKfyN~KKl@TY
    z#*Z{NSL+hy&~L0j8m)FS6CO>mXHI}j>(-o50pY-2Oa_as1E>6n&z2@
    z=($>3e!pJZ*|~{LXs1jzpSN|Dol>7rKqNvHp
    zeI$KjIj1
    zH|QkK$<^^I92ICrO7Tx*L&ueQ)mk5ZsJ4>VXq-b+_H!)xD>l^dC?5{L0Ad}r0x`rX
    z?T+w>*r;0UW5n%fGjRuwNj7|Y78(qxq)s@>)ua6U%p~QLGKPNO+vQl+%B2}wQ<7#n
    zD-$UO1jvi%eRCBw3G^7r5o3LDoCUeb-SFnt8W_Hnwu)F_?9lq+D}WMl6mg=wH9i(2
    z!3heZ4!19}uhA+EopLVu#S&Kgds^FE$`90CgJBn|29)DkmSeZ0s`fPWFM9<#K;|(C
    z%@V+V{C@JjZHaQ$(_E{nRw&tOoO%i!z~%UQkjKe5_@c5>xMQ0f>Rr|zi_+RwL}hhS
    z`%UI{QU482c(D*eRw3r4$Z>n+G5a)dn!
    z-AA0V4PuJ5c#T!&Q1$sdwTXTm&Q`~VQk7MuweybyTU}S^|78$N5uPa&@kU*(91y;#
    zthC?s!5t0%h#|@bbpbs*#EW~SB`u~TomFbXJK3j&%d{8xTzi&RrT5sE1d|&5=sv&_
    z*k~x*Y5{aHri}aYx
    z*sLdH3cz4_)N9Zjs-b}>Pc_G;0#((dz--Kq{T$3Dsw-89&De8#hG50|!q>r@NQN@S
    z-j*JZ@&0O|!-=Vs6TEBRhg5OZ!~yDU@Hx3pTAn^2e2z=a;K<*6AB7bdsx<)K
    zkfpZg+fCn~HDsrc0ZPMz>5e&MoIEejK2&#Qesn`Ufnc)Bk;Ol%Jlx=Dhaf`eu
    zU9#suSCDe(yNnUY3(rct)F5yz1Q2mnV5YW|H&aRCBX)HuAuFZNr9n!f8xibk7hg9c
    znP{qfPZa#uD_?^?&pRf_&NI>j2OBqbkxFI8syZ4HPpi(7{_F;_0ntiXFPt$5
    zYQv-)A_7m8^Z4Fs9x|e!E~N{f;^URBV%5yqN`F2A*~?yqXUSiDyU}rM4)nkUtEC|lW
    z8VDicwn5D3gf~iDLqz$$DbVn3U9?1+SX{@cF3
    zwwh#L+vdXGQLWenbx7E+=2*sC973uD$_LOIz&T5EgS+&Km>76x-6ffAlZZ=LB)h{%
    z3<@Q$!VW7F-T8qPLN5HPxHR^QWRdFl<|qw}w2rIFMj}IOtp@BXpwVcqmz9(TyOj3Q
    zH5XxcsUl6wg)5LN5XS!sK6fuhnyO2Et+8bQgQoZjfis}!9dAL%op@bxq})9%74Jp(
    z0-yM5#;kG@0!lBC+Q>WX9`^_wrDPQMjH{x)i{->|@?7y#pa&2xy(T#MW=K1&x-ebT
    zB?>*Kcj6b2cY_hA32i0*Yp%gNj45@qWQbV6oI`%HK2f6s$@osRrfsn`DBts(HBTs%
    z3NPSKeY1&$`7aci%k}oLq+y@5Sy;ZpYl+|lY5@{q{e_qSJrU31hMbZ7m$i%Wp`JG8|jOW}9Lm+^PTR-vvsm-|31QRy
    z%}yqQX4`)0OM*KyNG){lPIFPw${cJFJfaau9wo=46~3zIEqsxFI9N;_6O!%omFB6Z
    z@fIi~EktsdZ;Dx6VQ>f{r0l>DcN=Lv+OuhF@g-v`I1cNCepF-0xVqji>?q
    znAe5#k*jUf&~9W5a!f15dm{1V1XjR5V;;&1pOmZEhSL+-14gDpy21JF5}w2SgGNhN
    zg}t^W_UeE6hY-iWn5*Laev8800d`66MJ!eRBB>V)c
    zt^9{}rEVPm96`Q;wec&v8T(RTvL1)rhKaMEp=UbL0o<{VDU!V1&IkfK@{aEs%SzDj23n`;c8Os-VFxNWZRt}m93p8fxbg^msE;ZEBNMD-F{w3Ss3gs_#DEkCKk;dXEayer5YYH!1vv#FA_y+6c>PGD<
    zP}e;of0|eZ%k+67!tdJ9*%wbu8-oxsESsapa-`q#4W^5il
    zF?&3?zZP;8-A!D>{w4C{DcCyozu-`;!@+4(Q{cJV!#jd$>LjHLHrjZ94bTRW(`a4H
    zc2x$dA~a-9t*I-~h*2AOA@2Cp{jsn?K0+D$B4xx|ejM*HKqcwxGK
    z2z(T>p^rt(^B$@Wr2>y&L$w2p5|Xnjpg{x@Ocjj;M`6BE$MjME3T7$>eU-+D^TZ!|
    zjI>UU!V2+R^|-((BdKXbZ()$U24BeKE2^}YEwYkitooY118B@TESY>j<_Dcfq-IrG
    z@88cP6Pv|s(_ZZhz777_HBb@t*Mgm{X?X4@g14p5wttPTc?tY1R2U$%n}vTeP>DsZ
    zf!~6rh0uaU@MrU^|JRoN2}EICrBwmDit(u-l|ZSd)|%js!jVKB`$}j^)7oSkgMd9+
    z$;UdYE#jl_BDe%S>bhmnyd%ZS@W1d{d3p3$xGKLkWi{~9mW%r^&d}bT#om_~+Z+W^
    z64)wm7JNuJ%GZ={aSMg-fi~En2H&tQR4Z|;bw6^^Kt|284AY*PL7|q^2>eK7kuN#6
    zFA2Cfh`EOOY9Nw5jPb
    zQV&P*-g4)-bHE6HTYm)-8@z}&(z1P*;5b8H!C8J&{&8;-kC*-m)YlG5$=YeGH948+
    zu8gpk1aVG==a~b+(cbL_bFhHRRHtM2=sto=2)kg?qYGR#cr0^V_>FZCS|Cqzk3h16
    zYLJlTkXu4u)vE9ixl2YAd4by~W)k)AA#6i&B7Hd;~Ht&N1Ze!lkJdA!W|$VUWq?NS4hd!AhnS(v7E0ivDD|*%R9{V
    zkfzv2Vk@$qnTCEXr+`7?E93|8jd_v(O55j-h3F?Zg`H1*WPamQh)%+2JYd?3ZBR0Z
    z^WHzvhti$=k+u$Ofp>*3A@D8Gmh4D)rPkVc;H=Qd%VRwJQq6ZS#x98!+G%6`vx$${
    zSp$wCI-y@JhvARuv05A+ttOIRs66iv#*QV4_({k?L&7OG9_#Otd|`T;yCE}QSP%Q5
    zCd#ggL}d%s1$|ojxA~d;TE#&NbC+-d9kUwYdC(tfGh~5qo1m1n+JEFTl=NS>U$#{x
    zZlSt7#@0r8hAtz4&1?K$nWkuzSk4yE7XXF&Db*^y;Tj1>#Z=l+9R>b{KR1sSUs7Sf
    zf;6YTiVxT_<5NfFUUXluoADI6wf0QVYNHkbI5ms?Sy>=;(Z*ZS!5jE{-v#I#I#U{s
    zz6XuOAs`>0?VGK>CxRJ+Xc8QT#&Y@OtLojL3_>T`sLSKVgCEfK9Ir(S6Qp?Id!mv)
    z2hZ`}B)SXx*=g&)hNlRf)p73U{)53ia)h!$Di5BMpESB@@PJ2ZwdGK7q?Vq}o7+S6
    zIL2$oY-$(eFJK~k4dm2Up8H}A#H>%&CXv;_)kexg9efT3W2?mO)->uf`N}(8io9W2
    z`SK8GK6#W#fI9i&=%2XJ_$s0dQuw=}W6UP-IPn4D4PwG5vAcA{n7Dm!T8o&G?+xPQ
    zE~sP1zwkuINVJ;Ws-C9b8g9y3#P>$}o{eb`tPm%}rUO$-!s=?WQF?FJMXeAm^*51n
    zp%sCB#D3F8=2xME8YfRx=K=GmS_1t9!!zCb6(>^7nyh>WVpk|aCeiB4e
    zOG+IPvCs?xWoDS=&YioQduISqG%Z}h%!C%VH%T*5%O&;I%IH0R#QB`(Jm>j6-(XwD
    zM)@nESe&30(MSAgYIX7-E;F_}fPpaa1b*3mk`YLABp)U!9df>!-;fRB^$crO%Me8d
    ze~VB$uRH|n^i=*d@_(k8YRJT<&$NiDnJ8O1aTy*4FF1hfHTOb%7_5}0gE}}gE*gd}cGUTr_1#&hyG#t(Lhkn{Sm%GNt1p|i+Ajc
    z(x?2Tx<%}Eb#dgjKQ-f|_8~b*BP`v4t!g!v8+vK{DV0Rsa=S`MMn?np4cl#Y!Wr<6
    zzZ2WVmgr-dS=us_m-suYpz%yUr+yyvxQ~VRvw5TwTo-4`wc<`kKl6?lZ>f{wz_)M`w7R!(xm-W!%6J*|;QCCD
    zpqX*duc2Rp4*3!hYkegpHa(BdwzMMqD1cvRxJtz8tMyLnvbL4%biR!3-aUuPar7Ia
    zV5=*Q?iYgWZMFmHZUq{vi{NYOE-MDIwS#^$tP5Ce8|1Z38TRp7bR@^~I<(1mFkN{Z
    z-$>PXFTxkewZsyt+ISOeA+B&9>lpcaHo^M)eOh0{6NPQBOfTX>KdOhMQYPFQ9JgGRGrvcpiDX>
    zydjU1+{#cjS$sQkMmQ(=^b5#(-y0r9?xYKua=Y8o4<^u8?bTGDxkh~3an%~n=ScyH)-oFG03tru@C&Sm1>x5>fO?8XQ+=rS@?*S!?LVXa%oGzC0jQf72KxY7fQzz
    zqB~g(#h{E*zI~kV5_gxN8gjPBxx(&Y;S{1I)ZpX1WlCQA4E6`Sn*52LC|wE1NdL1|
    z>UHuLXy$Y~>?E^c1^E@SKqkUE@jEt_oGlGPf-M7+scNt?JRLt6ei+zBZ6%g7HP&6m
    z%j`Psnv{vMuxrl=|DQrZ{buuw%x#t)63Ma@qZY9@xDNdQ)yovpCy|qqMbA@e
    zZO_xIaOj(_%!LE_Rj9UQHAShEH%Ld6dL`Am&uWtroENQsFcs<|x8HYIto45Fv%~M*
    z0cw+4z%GP&^j@`?K1o(1QF9G3EopxGH`H;gSiQ!6Nlm3&a2b_8@9*9xen+L?Nz#+@
    zJt=>r!MjY(rAP3SgOhCO%3bqF&p=bPEljT_c-3N?>@6XkJB#oSuwqFF4AKHp9~P7w
    zn|p=V<*24N>Ys0_ln1MKK$QAaV=6}@jdCe?mMY`2)2h&I^?|1~^g&ZQ4#MxaU+|Y9
    zo1>H(hg%zC4MP-h?7LFDFqexK(Y9+$QldV
    zymf3N713@|0W4Zgr{|@Oq1&ij@X&M^7^yC|Mmx!y!XvKK7C}=gD}s}y!In~SDcxsT
    z5Jdets!J&M%nr2ElZi>`3AMXIXHn9pswsx8kQ~OfLXGthyd(Z00!H;FDaV^N>9C&Y
    zFAe@%!A=<^PU_T4A(50by@E_QRw%5OfFC$lu%9J~IU8ddEiL{K>Qj?VR
    z#1wB8s!Big+yH_X?Z=a@8soJq%nA7d{;qij`81=eB>B8;CfrOPt$QCo%QB&T&A*91
    z6NX5S5HTlWr@f0znbLAI;S%IWb)A;2(ofPSWFi)8zDG{*4yTsWy>hO)8Sl4RpkHnf
    zp94=2Yl)vtJ=mqxI}t2~-v{D45HyWX@1GnEhBZ60W|$ox*Tw3Ng{w
    z19Q}(k1IlFm>%#Iy#T#r8Tud7o3;k#7
    z^wW5}@GYGRN^BRY?}RRQQEC#bac6*A=~u>nW6rKo9Pf?|F_=`Ruh|I`*x_Se0Vwhv|M!L+9N;
    P3Yl!{g%Twjp7H%3hOI(i
    
    literal 0
    HcmV?d00001
    
    diff --git a/wasted/gtable_to_json.py b/wasted/gtable_to_json.py
    new file mode 100644
    index 0000000000..35282dd8a4
    --- /dev/null
    +++ b/wasted/gtable_to_json.py
    @@ -0,0 +1,18 @@
    +with open("sheet.csv", "r") as f:
    +    lines = f.readlines()
    +
    +keys = lines[0].strip().split(",")
    +dd = {}
    +print(keys)
    +for line in lines[1:]:
    +    values = line.strip().split(",")
    +    assert len(values) == len(keys)
    +    model_name = values[0]
    +    dd[model_name] = {}
    +    for k, v in zip(keys[1:], values[1:]):
    +        dd[model_name][k] = v
    +
    +import json
    +
    +with open("model_info.json", "w+") as f:
    +    json.dump(dd, f)
    diff --git a/wasted/load_np.py b/wasted/load_np.py
    new file mode 100644
    index 0000000000..7b92a7626e
    --- /dev/null
    +++ b/wasted/load_np.py
    @@ -0,0 +1,56 @@
    +import numpy as np
    +
    +datas = []
    +pre_x = None
    +
    +
    +def concat_qkv(datas):
    +    qs = []
    +    ks = []
    +    vs = []
    +    for d in datas:
    +        print(d.shape)
    +        qkv = np.split(d, 3, axis=-1)
    +        q = qkv[0]
    +        k = qkv[1]
    +        v = qkv[2]
    +        qs.append(q)
    +        ks.append(k)
    +        vs.append(v)
    +
    +    result = qs + ks + vs
    +    return np.concatenate(result, -1)
    +
    +
    +for i in range(4):
    +    x = np.load(f"disco_test/{i}.npy")
    +    datas.append(x)
    +    sum_result = np.sum(datas, axis=0)
    +    mean_result = np.mean(datas, axis=0)
    +    # concat_result = concat_qkv(datas)
    +    concat_result = np.concatenate(datas, axis=-1)
    +
    +    # print(i)
    +    # print(x)
    +    # print("====")
    +    # if pre_x is None or not np.allclose(x, pre_x):
    +    #     print(i)
    +    #     print(x)
    +    pre_x = x
    +
    +std_x = np.load("disco_test/single.npy")
    +
    +
    +result = sum_result
    +# print(datas[0])
    +# print(sum_result)
    +# print(concat_result)
    +# print(concat_result.shape)
    +# print(std_x.shape)
    +print(result)
    +print(std_x)
    +# print(concat_result - std_x)
    +print(np.max(result - std_x))
    +# print(np.sum((concat_result - std_x) > 0))
    +# print(np.argmax(concat_result - std_x))
    +print(np.allclose(result, std_x, atol=1e-1))
    diff --git a/wasted/model_info.json b/wasted/model_info.json
    new file mode 100644
    index 0000000000..6d2f7409ea
    --- /dev/null
    +++ b/wasted/model_info.json
    @@ -0,0 +1 @@
    +{"Llama-2-7b-hf": {"Model URL": "https://huggingface.co/meta-llama/Llama-2-7b-hf", "Model Type": "llama", "Context Window": "4096", "Conversation Template": "LM", "q3f16_1": "HF://junrushao/Llama-2-7b-hf-q3f16_1-MLC", "q4f16_1": "HF://junrushao/Llama-2-7b-hf-q4f16_1-MLC", "q4f32_1": "HF://junrushao/Llama-2-7b-hf-q4f32_1-MLC"}, "Llama-2-13b-hf": {"Model URL": "https://huggingface.co/meta-llama/Llama-2-13b-hf", "Model Type": "llama", "Context Window": "4096", "Conversation Template": "LM", "q3f16_1": "HF://junrushao/Llama-2-13b-hf-q3f16_1-MLC", "q4f16_1": "HF://junrushao/Llama-2-13b-hf-q4f16_1-MLC", "q4f32_1": "HF://junrushao/Llama-2-13b-hf-q4f32_1-MLC"}, "Llama-2-70b-hf": {"Model URL": "https://huggingface.co/meta-llama/Llama-2-70b-hf", "Model Type": "llama", "Context Window": "4096", "Conversation Template": "LM", "q3f16_1": "HF://junrushao/Llama-2-70b-hf-q3f16_1-MLC", "q4f16_1": "HF://junrushao/Llama-2-70b-hf-q4f16_1-MLC", "q4f32_1": "HF://junrushao/Llama-2-70b-hf-q4f32_1-MLC"}, "Llama-2-7b-chat-hf": {"Model URL": "https://huggingface.co/meta-llama/Llama-2-7b-chat-hf", "Model Type": "llama", "Context Window": "4096", "Conversation Template": "llama-2", "q3f16_1": "HF://junrushao/Llama-2-7b-chat-hf-q3f16_1-MLC", "q4f16_1": "HF://junrushao/Llama-2-7b-chat-hf-q4f16_1-MLC", "q4f32_1": "HF://junrushao/Llama-2-7b-chat-hf-q4f32_1-MLC"}, "Llama-2-13b-chat-hf": {"Model URL": "https://huggingface.co/meta-llama/Llama-2-13b-chat-hf", "Model Type": "llama", "Context Window": "4096", "Conversation Template": "llama-2", "q3f16_1": "HF://junrushao/Llama-2-13b-chat-hf-q3f16_1-MLC", "q4f16_1": "HF://junrushao/Llama-2-13b-chat-hf-q4f16_1-MLC", "q4f32_1": "HF://junrushao/Llama-2-13b-chat-hf-q4f32_1-MLC"}, "Llama-2-70b-chat-hf": {"Model URL": "https://huggingface.co/meta-llama/Llama-2-70b-chat-hf", "Model Type": "llama", "Context Window": "4096", "Conversation Template": "llama-2", "q3f16_1": "HF://junrushao/Llama-2-70b-chat-hf-q3f16_1-MLC", "q4f16_1": "HF://junrushao/Llama-2-70b-chat-hf-q4f16_1-MLC", "q4f32_1": "HF://junrushao/Llama-2-70b-chat-hf-q4f32_1-MLC"}, "CodeLlama-7b-hf": {"Model URL": "https://huggingface.co/codellama/CodeLlama-7b-hf", "Model Type": "llama", "Context Window": "16384", "Conversation Template": "LM", "q3f16_1": "HF://junrushao/CodeLlama-7b-hf-q3f16_1-MLC", "q4f16_1": "HF://junrushao/CodeLlama-7b-hf-q4f16_1-MLC", "q4f32_1": "HF://junrushao/CodeLlama-7b-hf-q4f32_1-MLC"}, "CodeLlama-13b-hf": {"Model URL": "https://huggingface.co/codellama/CodeLlama-13b-hf", "Model Type": "llama", "Context Window": "16384", "Conversation Template": "LM", "q3f16_1": "HF://junrushao/CodeLlama-13b-hf-q3f16_1-MLC", "q4f16_1": "HF://junrushao/CodeLlama-13b-hf-q4f16_1-MLC", "q4f32_1": "HF://junrushao/CodeLlama-13b-hf-q4f32_1-MLC"}, "CodeLlama-34b-hf": {"Model URL": "https://huggingface.co/codellama/CodeLlama-34b-hf", "Model Type": "llama", "Context Window": "16384", "Conversation Template": "LM", "q3f16_1": "HF://junrushao/CodeLlama-34b-hf-q3f16_1-MLC", "q4f16_1": "HF://junrushao/CodeLlama-34b-hf-q4f16_1-MLC", "q4f32_1": "HF://junrushao/CodeLlama-34b-hf-q4f32_1-MLC"}, "CodeLlama-7b-Instruct-hf": {"Model URL": "https://huggingface.co/codellama/CodeLlama-7b-Instruct-hf", "Model Type": "llama", "Context Window": "16384", "Conversation Template": "codellama_instruct", "q3f16_1": "HF://junrushao/CodeLlama-7b-Instruct-hf-q3f16_1-MLC", "q4f16_1": "HF://junrushao/CodeLlama-7b-Instruct-hf-q4f16_1-MLC", "q4f32_1": "HF://junrushao/CodeLlama-7b-Instruct-hf-q4f32_1-MLC"}, "CodeLlama-13b-Instruct-hf": {"Model URL": "https://huggingface.co/codellama/CodeLlama-13b-Instruct-hf", "Model Type": "llama", "Context Window": "16384", "Conversation Template": "codellama_instruct", "q3f16_1": "HF://junrushao/CodeLlama-13b-Instruct-hf-q3f16_1-MLC", "q4f16_1": "HF://junrushao/CodeLlama-13b-Instruct-hf-q4f16_1-MLC", "q4f32_1": "HF://junrushao/CodeLlama-13b-Instruct-hf-q4f32_1-MLC"}, "CodeLlama-34b-Instruct-hf": {"Model URL": "https://huggingface.co/codellama/CodeLlama-34b-Instruct-hf", "Model Type": "llama", "Context Window": "16384", "Conversation Template": "codellama_instruct", "q3f16_1": "HF://junrushao/CodeLlama-34b-Instruct-hf-q3f16_1-MLC", "q4f16_1": "HF://junrushao/CodeLlama-34b-Instruct-hf-q4f16_1-MLC", "q4f32_1": "HF://junrushao/CodeLlama-34b-Instruct-hf-q4f32_1-MLC"}, "CodeLlama-7b-Python-hf": {"Model URL": "https://huggingface.co/codellama/CodeLlama-7b-Python-hf", "Model Type": "llama", "Context Window": "16384", "Conversation Template": "codellama_instruct", "q3f16_1": "HF://junrushao/CodeLlama-7b-Python-hf-q3f16_1-MLC", "q4f16_1": "HF://junrushao/CodeLlama-7b-Python-hf-q4f16_1-MLC", "q4f32_1": "HF://junrushao/CodeLlama-7b-Python-hf-q4f32_1-MLC"}, "CodeLlama-13b-Python-hf": {"Model URL": "https://huggingface.co/codellama/CodeLlama-13b-Python-hf", "Model Type": "llama", "Context Window": "16384", "Conversation Template": "codellama_instruct", "q3f16_1": "HF://junrushao/CodeLlama-13b-Python-hf-q3f16_1-MLC", "q4f16_1": "HF://junrushao/CodeLlama-13b-Python-hf-q4f16_1-MLC", "q4f32_1": "HF://junrushao/CodeLlama-13b-Python-hf-q4f32_1-MLC"}, "CodeLlama-34b-Python-hf": {"Model URL": "https://huggingface.co/codellama/CodeLlama-34b-Python-hf", "Model Type": "llama", "Context Window": "16384", "Conversation Template": "codellama_instruct", "q3f16_1": "HF://junrushao/CodeLlama-34b-Python-hf-q3f16_1-MLC", "q4f16_1": "HF://junrushao/CodeLlama-34b-Python-hf-q4f16_1-MLC", "q4f32_1": "HF://junrushao/CodeLlama-34b-Python-hf-q4f32_1-MLC"}, "Mistral-7B-Instruct-v0.2": {"Model URL": "https://huggingface.co/mistralai/Mistral-7B-Instruct-v0.2", "Model Type": "mistral", "Context Window": "32768", "Conversation Template": "mistral_default", "q3f16_1": "HF://junrushao/Mistral-7B-Instruct-v0.2-q3f16_1-MLC", "q4f16_1": "HF://junrushao/Mistral-7B-Instruct-v0.2-q4f16_1-MLC", "q4f32_1": "HF://junrushao/Mistral-7B-Instruct-v0.2-q4f32_1-MLC"}, "NeuralHermes-2.5-Mistral-7B": {"Model URL": "https://huggingface.co/mlabonne/NeuralHermes-2.5-Mistral-7B", "Model Type": "mistral", "Context Window": "32768", "Conversation Template": "neural_hermes_mistral", "q3f16_1": "HF://junrushao/NeuralHermes-2.5-Mistral-7B-q3f16_1-MLC", "q4f16_1": "HF://junrushao/NeuralHermes-2.5-Mistral-7B-q4f16_1-MLC", "q4f32_1": "HF://junrushao/NeuralHermes-2.5-Mistral-7B-q4f32_1-MLC"}, "OpenHermes-2-Mistral-7B": {"Model URL": "https://huggingface.co/teknium/OpenHermes-2-Mistral-7B", "Model Type": "mistral", "Context Window": "32768", "Conversation Template": "open_hermes_mistral", "q3f16_1": "HF://junrushao/OpenHermes-2-Mistral-7B-q3f16_1-MLC", "q4f16_1": "HF://junrushao/OpenHermes-2-Mistral-7B-q4f16_1-MLC", "q4f32_1": "HF://junrushao/OpenHermes-2-Mistral-7B-q4f32_1-MLC"}, "WizardMath-7B-V1.1": {"Model URL": "https://huggingface.co/WizardLM/WizardMath-7B-V1.1", "Model Type": "mistral", "Context Window": "32768", "Conversation Template": "wizard_coder_or_math", "q3f16_1": "HF://junrushao/WizardMath-7B-V1.1-q3f16_1-MLC", "q4f16_1": "HF://junrushao/WizardMath-7B-V1.1-q4f16_1-MLC", "q4f32_1": "HF://junrushao/WizardMath-7B-V1.1-q4f32_1-MLC"}, "RedPajama-INCITE-Chat-3B-v1": {"Model URL": "https://huggingface.co/togethercomputer/RedPajama-INCITE-Chat-3B-v1", "Model Type": "gpt_neox", "Context Window": "2048", "Conversation Template": "redpajama_chat", "q3f16_1": "HF://junrushao/RedPajama-INCITE-Chat-3B-v1-q3f16_1-MLC", "q4f16_1": "HF://junrushao/RedPajama-INCITE-Chat-3B-v1-q4f16_1-MLC", "q4f32_1": "HF://junrushao/RedPajama-INCITE-Chat-3B-v1-q4f32_1-MLC"}, "phi-2": {"Model URL": "https://huggingface.co/microsoft/phi-2", "Model Type": "phi-msft", "Context Window": "2048", "Conversation Template": "phi-2", "q3f16_1": "HF://junrushao/phi-2-q3f16_1-MLC", "q4f16_1": "HF://junrushao/phi-2-q4f16_1-MLC", "q4f32_1": "HF://junrushao/phi-2-q4f32_1-MLC"}, "phi-1_5": {"Model URL": "https://huggingface.co/microsoft/phi-1_5", "Model Type": "phi-msft", "Context Window": "2048", "Conversation Template": "phi-2", "q3f16_1": "HF://junrushao/phi-1_5-q3f16_1-MLC", "q4f16_1": "HF://junrushao/phi-1_5-q4f16_1-MLC", "q4f32_1": "HF://junrushao/phi-1_5-q4f32_1-MLC"}, "gpt2": {"Model URL": "https://huggingface.co/gpt2", "Model Type": "gpt2", "Context Window": "1024", "Conversation Template": "gpt2", "q3f16_1": "HF://junrushao/gpt2-q3f16_1-MLC", "q4f16_1": "HF://junrushao/gpt2-q4f16_1-MLC", "q4f32_1": "HF://junrushao/gpt2-q4f32_1-MLC"}, "gpt2-medium": {"Model URL": "https://huggingface.co/gpt2-medium", "Model Type": "gpt2", "Context Window": "1024", "Conversation Template": "gpt2", "q3f16_1": "HF://junrushao/gpt2-medium-q3f16_1-MLC", "q4f16_1": "HF://junrushao/gpt2-medium-q4f16_1-MLC", "q4f32_1": "HF://junrushao/gpt2-medium-q4f32_1-MLC"}, "vicuna-7b-v1.5": {"Model URL": "https://huggingface.co/lmsys/vicuna-7b-v1.5", "Model Type": "llama", "Context Window": "4096", "Conversation Template": "vicuna_v1.1", "q3f16_1": "HF://junrushao/vicuna-7b-v1.5-q3f16_1-MLC", "q4f16_1": "HF://junrushao/vicuna-7b-v1.5-q4f16_1-MLC", "q4f32_1": "HF://junrushao/vicuna-7b-v1.5-q4f32_1-MLC"}, "WizardLM-7B-V1.0": {"Model URL": "https://huggingface.co/WizardLM/WizardLM-7B-V1.0", "Model Type": "llama", "Context Window": "2048", "Conversation Template": "vicuna_v1.1", "q3f16_1": "HF://junrushao/WizardLM-7B-V1.0-q3f16_1-MLC", "q4f16_1": "HF://junrushao/WizardLM-7B-V1.0-q4f16_1-MLC", "q4f32_1": "HF://junrushao/WizardLM-7B-V1.0-q4f32_1-MLC"}, "WizardLM-13B-V1.1": {"Model URL": "https://huggingface.co/WizardLM/WizardLM-13B-V1.1", "Model Type": "llama", "Context Window": "2048", "Conversation Template": "vicuna_v1.1", "q3f16_1": "HF://junrushao/WizardLM-13B-V1.1-q3f16_1-MLC", "q4f16_1": "HF://junrushao/WizardLM-13B-V1.1-q4f16_1-MLC", "q4f32_1": "HF://junrushao/WizardLM-13B-V1.1-q4f32_1-MLC"}, "WizardLM-30B-V1.0": {"Model URL": "https://huggingface.co/WizardLM/WizardLM-30B-V1.0", "Model Type": "llama", "Context Window": "2048", "Conversation Template": "vicuna_v1.1", "q3f16_1": "HF://junrushao/WizardLM-30B-V1.0-q3f16_1-MLC", "q4f16_1": "HF://junrushao/WizardLM-30B-V1.0-q3f16_1-MLC", "q4f32_1": "HF://junrushao/WizardLM-30B-V1.0-q4f32_1-MLC"}, "WizardLM-70B-V1.0": {"Model URL": "https://huggingface.co/WizardLM/WizardLM-70B-V1.0", "Model Type": "llama", "Context Window": "2048", "Conversation Template": "vicuna_v1.1", "q3f16_1": "HF://junrushao/WizardLM-70B-V1.0-q3f16_1-MLC", "q4f16_1": "HF://junrushao/WizardLM-70B-V1.0-q4f16_1-MLC", "q4f32_1": "HF://junrushao/WizardLM-70B-V1.0-q4f32_1-MLC"}, "WizardCoder-15B-V1.0": {"Model URL": "https://huggingface.co/WizardLM/WizardCoder-15B-V1.0", "Model Type": "gpt_bigcode", "Context Window": "8192", "Conversation Template": "wizard_coder_or_math", "q3f16_1": "HF://junrushao/WizardCoder-15B-V1.0-q3f16_1-MLC", "q4f16_1": "HF://junrushao/WizardCoder-15B-V1.0-q4f16_1-MLC", "q4f32_1": "HF://junrushao/WizardCoder-15B-V1.0-q4f32_1-MLC"}, "OpenOrca-Platypus2-13B": {"Model URL": "https://huggingface.co/Open-Orca/OpenOrca-Platypus2-13B", "Model Type": "llama", "Context Window": "4096", "Conversation Template": "llama-2", "q3f16_1": "HF://junrushao/OpenOrca-Platypus2-13B-q3f16_1-MLC", "q4f16_1": "HF://junrushao/OpenOrca-Platypus2-13B-q4f16_1-MLC", "q4f32_1": "HF://junrushao/OpenOrca-Platypus2-13B-q4f32_1-MLC"}, "Llama2-Chinese-7b-Chat": {"Model URL": "https://huggingface.co/FlagAlpha/Llama2-Chinese-7b-Chat", "Model Type": "llama", "Context Window": "4096", "Conversation Template": "llama-2", "q3f16_1": "HF://junrushao/Llama2-Chinese-7b-Chat-q3f16_1-MLC", "q4f16_1": "HF://junrushao/Llama2-Chinese-7b-Chat-q4f16_1-MLC", "q4f32_1": "HF://junrushao/Llama2-Chinese-7b-Chat-q4f32_1-MLC"}, "Llama2-Chinese-13b-Chat": {"Model URL": "https://huggingface.co/FlagAlpha/Llama2-Chinese-13b-Chat", "Model Type": "llama", "Context Window": "4096", "Conversation Template": "llama-2", "q3f16_1": "HF://junrushao/Llama2-Chinese-13b-Chat-q3f16_1-MLC", "q4f16_1": "HF://junrushao/Llama2-Chinese-13b-Chat-q4f16_1-MLC", "q4f32_1": "HF://junrushao/Llama2-Chinese-13b-Chat-q4f32_1-MLC"}, "llama2_7b_chat_uncensored": {"Model URL": "https://huggingface.co/georgesung/llama2_7b_chat_uncensored", "Model Type": "llama", "Context Window": "4096", "Conversation Template": "llama-default", "q3f16_1": "HF://junrushao/llama2_7b_chat_uncensored-q3f16_1-MLC", "q4f16_1": "HF://junrushao/llama2_7b_chat_uncensored-q4f16_1-MLC", "q4f32_1": "HF://junrushao/llama2_7b_chat_uncensored-q4f32_1-MLC"}, "WizardCoder-Python-34B-V1.0": {"Model URL": "https://huggingface.co/WizardLM/WizardCoder-Python-34B-V1.0", "Model Type": "llama", "Context Window": "16384", "Conversation Template": "wizard_coder_or_math", "q3f16_1": "HF://junrushao/WizardCoder-Python-34B-V1.0-q3f16_1-MLC", "q4f16_1": "HF://junrushao/WizardCoder-Python-34B-V1.0-q4f16_1-MLC", "q4f32_1": "HF://junrushao/WizardCoder-Python-34B-V1.0-q4f32_1-MLC"}, "open_llama_3b": {"Model URL": "https://huggingface.co/openlm-research/open_llama_3b", "Model Type": "llama", "Context Window": "2048", "Conversation Template": "llama-default", "q3f16_1": "HF://junrushao/open_llama_3b-q3f16_1-MLC", "q4f16_1": "HF://junrushao/open_llama_3b-q4f16_1-MLC", "q4f32_1": "HF://junrushao/open_llama_3b-q4f32_1-MLC"}, "open_llama_7b": {"Model URL": "https://huggingface.co/openlm-research/open_llama_7b", "Model Type": "llama", "Context Window": "2048", "Conversation Template": "llama-default", "q3f16_1": "HF://junrushao/open_llama_7b-q3f16_1-MLC", "q4f16_1": "HF://junrushao/open_llama_7b-q4f16_1-MLC", "q4f32_1": "HF://junrushao/open_llama_7b-q4f32_1-MLC"}, "open_llama_13b": {"Model URL": "https://huggingface.co/openlm-research/open_llama_13b", "Model Type": "llama", "Context Window": "2048", "Conversation Template": "llama-default", "q3f16_1": "HF://junrushao/open_llama_13b-q3f16_1-MLC", "q4f16_1": "HF://junrushao/open_llama_13b-q4f16_1-MLC", "q4f32_1": "HF://junrushao/open_llama_13b-q4f32_1-MLC"}, "YuLan-Chat-2-13b-fp16": {"Model URL": "https://huggingface.co/yulan-team/YuLan-Chat-2-13b-fp16", "Model Type": "llama", "Context Window": "8192", "Conversation Template": "llama-2", "q3f16_1": "HF://junrushao/YuLan-Chat-2-13b-fp16-q3f16_1-MLC", "q4f16_1": "HF://junrushao/YuLan-Chat-2-13b-fp16-q4f16_1-MLC", "q4f32_1": "HF://junrushao/YuLan-Chat-2-13b-fp16-q4f32_1-MLC"}, "dolly-v2-12b": {"Model URL": "https://huggingface.co/databricks/dolly-v2-12b", "Model Type": "gpt_neox", "Context Window": "2048", "Conversation Template": "dolly", "q3f16_1": "HF://junrushao/dolly-v2-12b-q3f16_1-MLC", "q4f16_1": "HF://junrushao/dolly-v2-12b-q4f16_1-MLC", "q4f32_1": "HF://junrushao/dolly-v2-12b-q4f32_1-MLC"}, "pythia-1.4b": {"Model URL": "https://huggingface.co/EleutherAI/pythia-1.4b", "Model Type": "gpt_neox", "Context Window": "2048", "Conversation Template": "LM", "q3f16_1": "HF://junrushao/pythia-1.4b-q3f16_1-MLC", "q4f16_1": "HF://junrushao/pythia-1.4b-q4f16_1-MLC", "q4f32_1": "HF://junrushao/pythia-1.4b-q4f32_1-MLC"}, "stablecode-instruct-alpha-3b": {"Model URL": "https://huggingface.co/stabilityai/stablecode-instruct-alpha-3b", "Model Type": "gpt_neox", "Context Window": "4096", "Conversation Template": "stablecode_instruct", "q3f16_1": "HF://junrushao/stablecode-instruct-alpha-3b-q3f16_1-MLC", "q4f16_1": "HF://junrushao/stablecode-instruct-alpha-3b-q4f16_1-MLC", "q4f32_1": "HF://junrushao/stablecode-instruct-alpha-3b-q4f32_1-MLC"}, "starcoder": {"Model URL": "https://huggingface.co/bigcode/starcoder", "Model Type": "gpt_bigcode", "Context Window": "8192", "Conversation Template": "LM", "q3f16_1": "HF://junrushao/starcoder-q3f16_1-MLC", "q4f16_1": "HF://junrushao/starcoder-q4f16_1-MLC", "q4f32_1": "HF://junrushao/starcoder-q4f32_1-MLC"}, "gpt_bigcode-santacoder": {"Model URL": "https://huggingface.co/bigcode/gpt_bigcode-santacoder", "Model Type": "gpt_bigcode", "Context Window": "2048", "Conversation Template": "LM", "q3f16_1": "HF://junrushao/gpt_bigcode-santacoder-q3f16_1-MLC", "q4f16_1": "HF://junrushao/gpt_bigcode-santacoder-q4f16_1-MLC", "q4f32_1": "HF://junrushao/gpt_bigcode-santacoder-q4f32_1-MLC"}}
    \ No newline at end of file
    diff --git a/wasted/phi_model_old.py b/wasted/phi_model_old.py
    new file mode 100644
    index 0000000000..8e520f6653
    --- /dev/null
    +++ b/wasted/phi_model_old.py
    @@ -0,0 +1,451 @@
    +"""
    +Implementation for Phi architecture.
    +TODO: add docstring
    +"""
    +import dataclasses
    +from typing import Any, Dict, Optional, Union
    +
    +from tvm import te, tir
    +from tvm.relax.frontend import nn
    +from tvm.relax.frontend.nn import Tensor, op
    +
    +from mlc_chat import op as op_ext
    +from mlc_chat.support import logging
    +from mlc_chat.support import tensor_parallel as tp
    +from mlc_chat.support.config import ConfigBase
    +from mlc_chat.support.style import bold
    +
    +logger = logging.getLogger(__name__)
    +
    +
    +@dataclasses.dataclass
    +class Phi1Config(ConfigBase):  # pylint: disable=too-many-instance-attributes
    +    """Configuration of the Phi-1/Phi-1.5 model."""
    +
    +    vocab_size: int = 51200
    +    hidden_size: int = 2048
    +    intermediate_size: int = 8192
    +    num_hidden_layers: int = 24
    +    num_attention_heads: int = 32
    +    layer_norm_eps: float = 1e-5
    +    position_embedding_base: int = 0
    +    partial_rotary_factor: float = 0.5
    +    num_key_value_heads: int = 0
    +    context_window_size: int = 0
    +    prefill_chunk_size: int = 0
    +    head_dim: int = 0
    +    tensor_parallel_shards: int = 1
    +    kwargs: Dict[str, Any] = dataclasses.field(default_factory=dict)
    +
    +    def __post_init__(self):
    +        if self.position_embedding_base == 0:
    +            if "rope_theta" in self.kwargs:
    +                self.position_embedding_base = self.kwargs.pop("rope_theta")
    +            else:
    +                self.position_embedding_base = 10000
    +        if self.context_window_size == 0:
    +            for name in ["max_position_embeddings", "max_sequence_length"]:
    +                if name in self.kwargs:
    +                    self.context_window_size = self.kwargs.pop(name)
    +                    logger.info(
    +                        "%s not found in config.json. Falling back to %s (%d)",
    +                        bold("context_window_size"),
    +                        bold(name),
    +                        self.context_window_size,
    +                    )
    +                    break
    +            else:
    +                raise ValueError(
    +                    "Unable to determine the maxmimum sequence length, because none of "
    +                    "`context_window_size`, `max_position_embeddings` or `max_sequence_length` is "
    +                    "provided in `config.json`."
    +                )
    +        if self.prefill_chunk_size == 0:
    +            self.prefill_chunk_size = self.context_window_size
    +        if self.prefill_chunk_size > self.context_window_size:
    +            self.prefill_chunk_size = self.context_window_size
    +        if self.num_key_value_heads == 0 or self.num_key_value_heads is None:
    +            self.num_key_value_heads = self.num_attention_heads
    +        if self.intermediate_size == 0 or self.intermediate_size is None:
    +            self.intermediate_size = 4 * self.hidden_size
    +        if self.head_dim == 0:
    +            self.head_dim = self.hidden_size // self.num_attention_heads
    +        assert self.head_dim * self.num_attention_heads == self.hidden_size
    +        assert self.num_attention_heads % self.num_key_value_heads == 0
    +
    +
    +@dataclasses.dataclass
    +class PhiConfig(ConfigBase):  # pylint: disable=too-many-instance-attributes
    +    """Configuration of the Phi-2 model."""
    +
    +    model_type: str  # "phi", "phi-msft", "mixformer-sequential"
    +    vocab_size: int = 51200
    +    n_positions: int = 2048
    +    n_embd: int = 2560
    +    n_layer: int = 32
    +    n_inner: int = 0
    +    n_head: int = 32
    +    rotary_dim: int = 32
    +    position_embedding_base: int = 0
    +    layer_norm_epsilon: float = 1e-5
    +    context_window_size: int = 0
    +    prefill_chunk_size: int = 0
    +    n_head_kv: int = 0
    +    head_dim: int = 0
    +    tensor_parallel_shards: int = 1
    +    kwargs: Dict[str, Any] = dataclasses.field(default_factory=dict)
    +
    +    def __post_init__(self):
    +        if self.position_embedding_base == 0:
    +            if "rope_theta" in self.kwargs:
    +                self.position_embedding_base = self.kwargs.pop("rope_theta")
    +            else:
    +                self.position_embedding_base = 10000
    +        if self.context_window_size == 0:
    +            for name in ["max_position_embeddings", "max_sequence_length"]:
    +                if name in self.kwargs:
    +                    self.context_window_size = self.kwargs.pop(name)
    +                    logger.info(
    +                        "%s not found in config.json. Falling back to %s (%d)",
    +                        bold("context_window_size"),
    +                        bold(name),
    +                        self.context_window_size,
    +                    )
    +                    break
    +            else:
    +                self.context_window_size = self.n_positions
    +                logger.info(
    +                    "%s not found in config.json. Falling back to %s (%d)",
    +                    bold("context_window_size"),
    +                    "n_positions",
    +                    self.context_window_size,
    +                )
    +        if self.prefill_chunk_size == 0:
    +            self.prefill_chunk_size = self.context_window_size
    +        if self.prefill_chunk_size > self.context_window_size:
    +            self.prefill_chunk_size = self.context_window_size
    +        if self.n_head_kv == 0 or self.n_head_kv is None:
    +            self.n_head_kv = self.n_head
    +        if self.n_inner == 0 or self.n_inner is None:
    +            self.n_inner = 4 * self.n_embd
    +        if self.head_dim == 0:
    +            self.head_dim = self.n_embd // self.n_head
    +        assert self.head_dim * self.n_head == self.n_embd
    +        assert self.n_head % self.n_head_kv == 0
    +
    +    @staticmethod
    +    def from_phi1(config: Phi1Config) -> "PhiConfig":
    +        "Build PhiConig from a Phi1Config."
    +        return PhiConfig(
    +            model_type="phi",
    +            vocab_size=config.vocab_size,
    +            n_positions=config.context_window_size,
    +            n_embd=config.hidden_size,
    +            n_layer=config.num_hidden_layers,
    +            n_inner=config.intermediate_size,
    +            n_head=config.num_attention_heads,
    +            rotary_dim=int(config.partial_rotary_factor * config.head_dim),
    +            position_embedding_base=config.position_embedding_base,
    +            layer_norm_epsilon=config.layer_norm_eps,
    +            context_window_size=config.context_window_size,
    +            prefill_chunk_size=config.prefill_chunk_size,
    +            n_head_kv=config.num_key_value_heads,
    +            head_dim=config.head_dim,
    +            tensor_parallel_shards=config.tensor_parallel_shards,
    +            kwargs=config.kwargs,
    +        )
    +
    +
    +# pylint: disable=invalid-name,missing-docstring
    +
    +
    +class PhiMLP(nn.Module):
    +    def __init__(self, config: PhiConfig):
    +        super().__init__()
    +        self.intermediate_size = config.n_inner // config.tensor_parallel_shards
    +        self.fc1 = nn.Linear(config.n_embd, self.intermediate_size)
    +        self.fc2 = nn.Linear(self.intermediate_size, config.n_embd)
    +
    +    def forward(self, hidden_states: Tensor):
    +        hidden_states = self.fc1(hidden_states)
    +        hidden_states = op.gelu(hidden_states, approximate="tanh")
    +        hidden_states = self.fc2(hidden_states)
    +
    +        return hidden_states
    +
    +
    +class PhiCrossAttention(nn.Module):
    +    def __init__(self, config: PhiConfig):  # pylint: disable=unused-argument
    +        super().__init__()
    +
    +    def forward(self, q: Tensor, k: Tensor, v: Tensor, attention_mask: Tensor):
    +        output = op_ext.attention(q, k, v, casual_mask=attention_mask, qk_dtype="float32")
    +        return output
    +
    +
    +class PhiMHA(nn.Module):  # pylint: disable=too-many-instance-attributes
    +    def __init__(self, config: PhiConfig):
    +        self.rope_theta = config.position_embedding_base
    +        self.rotary_dim = config.rotary_dim
    +        self.n_head = config.n_head // config.tensor_parallel_shards
    +        assert (
    +            config.n_head % config.tensor_parallel_shards == 0
    +        ), f"n_head({config.n_head}) must be divisible by tensor_parallel_shards"
    +        self.n_head_kv = config.n_head_kv // config.tensor_parallel_shards
    +        assert (
    +            config.n_head_kv % config.tensor_parallel_shards == 0
    +        ), f"n_head({config.n_head_kv}) must be divisible by tensor_parallel_shards"
    +        self.head_dim = config.head_dim
    +        op_size = self.head_dim * (self.n_head + 2 * self.n_head_kv)
    +        hidden_size = config.n_embd
    +
    +        self.Wqkv = nn.Linear(hidden_size, op_size, bias=True)
    +        self.out_proj = nn.Linear(self.n_head * self.head_dim, hidden_size, bias=True)
    +        self.inner_cross_attn = PhiCrossAttention(config)
    +        self.k_cache = nn.KVCache(config.context_window_size, [self.n_head_kv, self.head_dim])
    +        self.v_cache = nn.KVCache(config.context_window_size, [self.n_head_kv, self.head_dim])
    +
    +    def forward(self, x: Tensor, attention_mask: Tensor, total_seq_len: tir.Var):
    +        d, h_q, h_kv, t = self.head_dim, self.n_head, self.n_head_kv, total_seq_len
    +        b, s, _ = x.shape
    +        assert b == 1, "Only support batch size 1 at this moment."
    +        # Step 1. QKV Projection
    +        qkv = self.Wqkv(x)
    +        # if self.idx == 0:
    +        #     op.print_inter(qkv, self.tensor_parallel_shards)
    +        qkv = op.reshape(qkv, (b, s, h_q + h_kv + h_kv, d))
    +        # Step 2. Apply QK rotary embedding
    +        q, k, v = op_ext.llama_rope(qkv, t, self.rope_theta, h_q, h_kv, rotary_dim=self.rotary_dim)
    +        # if self.idx == 0:
    +        #     op.print_inter(q, self.tensor_parallel_shards)
    +        # Step 3. Query and update KVCache
    +        self.k_cache.append(op.squeeze(k, axis=0))
    +        self.v_cache.append(op.squeeze(v, axis=0))
    +        k = self.k_cache.view(t)
    +        v = self.v_cache.view(t)
    +        # Step 4. Compute softmax(Q @ K^T / sqrt(d)) @ V
    +        # if self.idx == 0:
    +        #     op.print_inter(k, self.tensor_parallel_shards)
    +        output = self.inner_cross_attn(q, k, v, attention_mask)
    +        # if self.idx == 0:
    +        #     op.print_inter(output, self.tensor_parallel_shards)
    +        # Step 5. Apply output projection
    +        self.out_proj.idx = self.idx
    +        return self.out_proj(output)
    +
    +
    +class PhiParallelBlock(nn.Module):
    +    def __init__(self, config: PhiConfig):
    +        super().__init__()
    +
    +        self.ln = nn.LayerNorm(config.n_embd, eps=config.layer_norm_epsilon)
    +        self.mixer = PhiMHA(config)
    +        self.mlp = PhiMLP(config)
    +
    +        def _set_tp():
    +            def _set(param, hint):
    +                param.attrs["shard_strategy"] = hint
    +
    +            hd = config.head_dim
    +            q = self.mixer.n_head * hd
    +            k = self.mixer.n_head_kv * hd
    +            v = self.mixer.n_head_kv * hd
    +            _set(
    +                self.mixer.Wqkv.weight,
    +                tp.ShardSingleDim("_shard_qkv_weight", segs=[q, k, v], dim=0),
    +            )
    +            _set(self.mixer.Wqkv.bias, tp.ShardSingleDim("_shard_qkv_bias", segs=[q, k, v], dim=0))
    +            _set(self.mixer.out_proj.weight, tp.ShardSingleDim("_shard_o_weight", dim=1))
    +            _set(self.mlp.fc1.weight, tp.ShardSingleDim("_shard_mlp_fc1_weight", dim=0))
    +            _set(self.mlp.fc1.bias, tp.ShardSingleDim("_shard_mlp_fc1_bias", dim=0))
    +            _set(self.mlp.fc2.weight, tp.ShardSingleDim("_shard_mlp_fc2_weight", dim=1))
    +
    +        self.tensor_parallel_shards = config.tensor_parallel_shards
    +        _set_tp()
    +
    +    def forward(self, hidden_states: Tensor, attention_mask: Tensor, total_seq_len: tir.Var):
    +        residual = hidden_states
    +        hidden_states = self.ln(hidden_states)
    +
    +        self.mixer.idx = self.idx
    +        self.mlp.idx = self.idx
    +        self.mixer.tensor_parallel_shards = self.tensor_parallel_shards
    +        self.mlp.tensor_parallel_shards = self.tensor_parallel_shards
    +
    +        # attn_outputs = (
    +        #     self.mixer(
    +        #         hidden_states,
    +        #         attention_mask,
    +        #         total_seq_len,
    +        #     )
    +        #     - self.mixer.out_proj.bias
    +        # )
    +
    +        # if self.idx == 0:
    +        #     op.print_inter(attn_outputs, self.tensor_parallel_shards)
    +
    +        # feed_forward_hidden_states = self.mlp(hidden_states) - self.mlp.fc2.bias
    +
    +        # if self.idx == 0:
    +        #     op.print_inter(feed_forward_hidden_states, self.tensor_parallel_shards)
    +
    +        with tp.shard_bias(self.mixer.out_proj, self.tensor_parallel_shards), tp.shard_bias(
    +            self.mlp.fc2, self.tensor_parallel_shards
    +        ):
    +            attn_outputs = self.mixer(
    +                hidden_states,
    +                attention_mask,
    +                total_seq_len,
    +            )
    +
    +            # if self.idx == 0:
    +            #     op.print_inter(attn_outputs, self.tensor_parallel_shards)
    +
    +            feed_forward_hidden_states = self.mlp(hidden_states)
    +
    +            # if self.idx == 0:
    +            #     op.print_inter(feed_forward_hidden_states, self.tensor_parallel_shards)
    +
    +        def _apply_parallel_residual(attn_out, mlp_out, residual):
    +            if self.tensor_parallel_shards > 1:
    +                return op.ccl_allreduce(
    +                    attn_out + mlp_out + residual / self.tensor_parallel_shards, "sum"
    +                )
    +            return attn_out + mlp_out + residual
    +
    +        hidden_states = _apply_parallel_residual(attn_outputs, feed_forward_hidden_states, residual)
    +
    +        # if self.tensor_parallel_shards > 1:
    +        # hidden_states = hidden_states + self.mixer.out_proj.bias + self.mlp.fc2.bias
    +
    +        # if self.idx == 0:
    +        #     op.print_inter(hidden_states, self.tensor_parallel_shards)
    +
    +        return hidden_states
    +
    +
    +class PhiCausalLMHead(nn.Module):
    +    def __init__(self, config: PhiConfig) -> None:
    +        super().__init__()
    +
    +        self.ln = nn.LayerNorm(config.n_embd, eps=config.layer_norm_epsilon)
    +        self.linear = nn.Linear(config.n_embd, "vocab_size")
    +
    +    def forward(self, hidden_states: Tensor):
    +        hidden_states = self.ln(hidden_states)
    +        logits = self.linear(hidden_states)
    +
    +        if logits.dtype != "float32":
    +            logits = logits.astype("float32")
    +        return logits
    +
    +
    +class PhiModel(nn.Module):
    +    def __init__(self, config: PhiConfig) -> None:
    +        super().__init__()
    +        self.embd = nn.Embedding("vocab_size", config.n_embd)
    +        self.h = nn.ModuleList([PhiParallelBlock(config) for i in range(config.n_layer)])
    +        self.tensor_parallel_shards = config.tensor_parallel_shards
    +
    +    def forward(self, input_ids: Tensor, total_seq_len: tir.Var, attention_mask: Tensor):
    +        if self.tensor_parallel_shards > 1:
    +            input_ids = op.ccl_broadcast_from_worker0(input_ids)
    +        hidden_states = self.embd(input_ids)
    +        # op.print_inter(hidden_states)
    +        for i, layer in enumerate(self.h):
    +            layer.idx = i
    +            hidden_states = layer(hidden_states, attention_mask, total_seq_len)
    +            # if i == 0:
    +            #     op.print_inter(hidden_states)
    +
    +        return hidden_states
    +
    +
    +class PhiForCausalLM(nn.Module):
    +    def __init__(self, config: Union[PhiConfig, Phi1Config]) -> None:
    +        super().__init__()
    +
    +        if isinstance(config, Phi1Config):
    +            config = PhiConfig.from_phi1(config)
    +
    +        self.transformer = PhiModel(config)
    +        self.lm_head = PhiCausalLMHead(config)
    +        self.dtype = "float32"
    +
    +    def to(self, dtype: Optional[str] = None):
    +        super().to(dtype=dtype)
    +        if dtype is not None:
    +            self.dtype = dtype
    +
    +    def forward(self, input_ids: Tensor, total_seq_len: tir.Var, attention_mask: Tensor):
    +        def _index(x: te.Tensor):  # x[:-1,:]
    +            b, s, d = x.shape
    +            return te.compute((b, 1, d), lambda i, _, k: x[i, s - 1, k], name="index")
    +
    +        hidden_states = self.transformer(input_ids, total_seq_len, attention_mask)
    +        hidden_states = op.tensor_expr_op(_index, name_hint="index", args=[hidden_states])
    +        lm_logits = self.lm_head(hidden_states)
    +
    +        return lm_logits
    +
    +    def prefill(self, inputs: Tensor, total_seq_len: tir.Var):
    +        def _attention_mask(batch_size, seq_len, total_seq_len):
    +            return te.compute(
    +                (batch_size, 1, seq_len, total_seq_len),
    +                lambda b, _, i, j: tir.if_then_else(
    +                    i < j - (total_seq_len - seq_len),
    +                    tir.min_value(self.dtype),
    +                    tir.max_value(self.dtype),
    +                ),
    +                name="attention_mask_prefill",
    +            )
    +
    +        batch_size, seq_len = inputs.shape
    +        attention_mask = op.tensor_expr_op(
    +            _attention_mask,
    +            name_hint="attention_mask_prefill",
    +            args=[batch_size, seq_len, total_seq_len],
    +        )
    +        return self.forward(inputs, total_seq_len, attention_mask)
    +
    +    def decode(self, inputs: Tensor, total_seq_len: tir.Var):
    +        batch_size, seq_len = inputs.shape
    +        attention_mask = op.full(
    +            shape=[batch_size, 1, seq_len, total_seq_len],
    +            fill_value=tir.max_value(self.dtype),
    +            dtype=self.dtype,
    +        )
    +        return self.forward(inputs, total_seq_len, attention_mask)
    +
    +    def softmax_with_temperature(self, logits: Tensor, temperature: Tensor):
    +        return op.softmax(logits / temperature, axis=-1)
    +
    +    def get_default_spec(self):
    +        batch_size = 1
    +        mod_spec = {
    +            "prefill": {
    +                "inputs": nn.spec.Tensor([batch_size, "seq_len"], "int32"),
    +                "total_seq_len": int,
    +                "$": {
    +                    "param_mode": "packed",
    +                    "effect_mode": "packed",
    +                },
    +            },
    +            "decode": {
    +                "inputs": nn.spec.Tensor([batch_size, 1], "int32"),
    +                "total_seq_len": int,
    +                "$": {
    +                    "param_mode": "packed",
    +                    "effect_mode": "packed",
    +                },
    +            },
    +            "softmax_with_temperature": {
    +                "logits": nn.spec.Tensor([1, 1, "vocab_size"], "float32"),
    +                "temperature": nn.spec.Tensor([], "float32"),
    +                "$": {
    +                    "param_mode": "none",
    +                    "effect_mode": "none",
    +                },
    +            },
    +        }
    +        return nn.spec.ModuleSpec.from_raw(mod_spec, self)
    diff --git a/wasted/sample_mlc_chat.py b/wasted/sample_mlc_chat.py
    new file mode 100644
    index 0000000000..b97c700b90
    --- /dev/null
    +++ b/wasted/sample_mlc_chat.py
    @@ -0,0 +1,27 @@
    +from mlc_chat import ChatConfig, ChatModule
    +from mlc_chat.callback import StreamToStdout
    +
    +# From the mlc-llm directory, run
    +# $ python examples/python/sample_mlc_chat.py
    +
    +# Create a ChatModule instance
    +# Multiple GPU
    +cm = ChatModule(
    +    # model="dist/llama-2-7b-chat-hf-q4f16_1/params",
    +    model="dist/llama-2-7b-chat-hf-q4f16_1-presharded-2gpu/params",
    +    chat_config=ChatConfig(tensor_parallel_shards=2)
    +    # model="dist/Mixtral-8x7B-Instruct-v0.1-q0f16/params",
    +    # chat_config=ChatConfig(tensor_parallel_shards=2),
    +)
    +
    +# Generate a response for a given prompt
    +output = cm.generate(
    +    prompt="What is the meaning of life?",
    +    progress_callback=StreamToStdout(callback_interval=2),
    +)
    +
    +# Print prefill and decode performance statistics
    +print(f"Statistics: {cm.stats()}\n")
    +
    +# Reset the chat module by
    +cm.reset_chat()
    diff --git a/wasted/sheet.csv b/wasted/sheet.csv
    new file mode 100644
    index 0000000000..be037c7d47
    --- /dev/null
    +++ b/wasted/sheet.csv
    @@ -0,0 +1,29 @@
    +Name (last piece of Model URL),Model URL,Model Type,Context Window,Conversation Template,q3f16_1,q4f16_1,q4f32_1
    +Llama-2-7b-hf,https://huggingface.co/meta-llama/Llama-2-7b-hf,llama,4096,LM,HF://junrushao/Llama-2-7b-hf-q3f16_1-MLC,HF://junrushao/Llama-2-7b-hf-q4f16_1-MLC,HF://junrushao/Llama-2-7b-hf-q4f32_1-MLC
    +Llama-2-13b-hf,https://huggingface.co/meta-llama/Llama-2-13b-hf,llama,4096,LM,HF://junrushao/Llama-2-13b-hf-q3f16_1-MLC,HF://junrushao/Llama-2-13b-hf-q4f16_1-MLC,HF://junrushao/Llama-2-13b-hf-q4f32_1-MLC
    +Llama-2-70b-hf,https://huggingface.co/meta-llama/Llama-2-70b-hf,llama,4096,LM,HF://junrushao/Llama-2-70b-hf-q3f16_1-MLC,HF://junrushao/Llama-2-70b-hf-q4f16_1-MLC,HF://junrushao/Llama-2-70b-hf-q4f32_1-MLC
    +Llama-2-7b-chat-hf,https://huggingface.co/meta-llama/Llama-2-7b-chat-hf,llama,4096,llama-2,HF://junrushao/Llama-2-7b-chat-hf-q3f16_1-MLC,HF://junrushao/Llama-2-7b-chat-hf-q4f16_1-MLC,HF://junrushao/Llama-2-7b-chat-hf-q4f32_1-MLC
    +Llama-2-13b-chat-hf,https://huggingface.co/meta-llama/Llama-2-13b-chat-hf,llama,4096,llama-2,HF://junrushao/Llama-2-13b-chat-hf-q3f16_1-MLC,HF://junrushao/Llama-2-13b-chat-hf-q4f16_1-MLC,HF://junrushao/Llama-2-13b-chat-hf-q4f32_1-MLC
    +Llama-2-70b-chat-hf,https://huggingface.co/meta-llama/Llama-2-70b-chat-hf,llama,4096,llama-2,HF://junrushao/Llama-2-70b-chat-hf-q3f16_1-MLC,HF://junrushao/Llama-2-70b-chat-hf-q4f16_1-MLC,HF://junrushao/Llama-2-70b-chat-hf-q4f32_1-MLC
    +CodeLlama-7b-hf,https://huggingface.co/codellama/CodeLlama-7b-hf,llama,16384,LM,HF://junrushao/CodeLlama-7b-hf-q3f16_1-MLC,HF://junrushao/CodeLlama-7b-hf-q4f16_1-MLC,HF://junrushao/CodeLlama-7b-hf-q4f32_1-MLC
    +CodeLlama-13b-hf,https://huggingface.co/codellama/CodeLlama-13b-hf,llama,16384,LM,HF://junrushao/CodeLlama-13b-hf-q3f16_1-MLC,HF://junrushao/CodeLlama-13b-hf-q4f16_1-MLC,HF://junrushao/CodeLlama-13b-hf-q4f32_1-MLC
    +CodeLlama-34b-hf,https://huggingface.co/codellama/CodeLlama-34b-hf,llama,16384,LM,HF://junrushao/CodeLlama-34b-hf-q3f16_1-MLC,HF://junrushao/CodeLlama-34b-hf-q4f16_1-MLC,HF://junrushao/CodeLlama-34b-hf-q4f32_1-MLC
    +CodeLlama-7b-Instruct-hf,https://huggingface.co/codellama/CodeLlama-7b-Instruct-hf,llama,16384,codellama_instruct,HF://junrushao/CodeLlama-7b-Instruct-hf-q3f16_1-MLC,HF://junrushao/CodeLlama-7b-Instruct-hf-q4f16_1-MLC,HF://junrushao/CodeLlama-7b-Instruct-hf-q4f32_1-MLC
    +CodeLlama-13b-Instruct-hf,https://huggingface.co/codellama/CodeLlama-13b-Instruct-hf,llama,16384,codellama_instruct,HF://junrushao/CodeLlama-13b-Instruct-hf-q3f16_1-MLC,HF://junrushao/CodeLlama-13b-Instruct-hf-q4f16_1-MLC,HF://junrushao/CodeLlama-13b-Instruct-hf-q4f32_1-MLC
    +CodeLlama-34b-Instruct-hf,https://huggingface.co/codellama/CodeLlama-34b-Instruct-hf,llama,16384,codellama_instruct,HF://junrushao/CodeLlama-34b-Instruct-hf-q3f16_1-MLC,HF://junrushao/CodeLlama-34b-Instruct-hf-q4f16_1-MLC,HF://junrushao/CodeLlama-34b-Instruct-hf-q4f32_1-MLC
    +CodeLlama-7b-Python-hf,https://huggingface.co/codellama/CodeLlama-7b-Python-hf,llama,16384,codellama_instruct,HF://junrushao/CodeLlama-7b-Python-hf-q3f16_1-MLC,HF://junrushao/CodeLlama-7b-Python-hf-q4f16_1-MLC,HF://junrushao/CodeLlama-7b-Python-hf-q4f32_1-MLC
    +CodeLlama-13b-Python-hf,https://huggingface.co/codellama/CodeLlama-13b-Python-hf,llama,16384,codellama_instruct,HF://junrushao/CodeLlama-13b-Python-hf-q3f16_1-MLC,HF://junrushao/CodeLlama-13b-Python-hf-q4f16_1-MLC,HF://junrushao/CodeLlama-13b-Python-hf-q4f32_1-MLC
    +CodeLlama-34b-Python-hf,https://huggingface.co/codellama/CodeLlama-34b-Python-hf,llama,16384,codellama_instruct,HF://junrushao/CodeLlama-34b-Python-hf-q3f16_1-MLC,HF://junrushao/CodeLlama-34b-Python-hf-q4f16_1-MLC,HF://junrushao/CodeLlama-34b-Python-hf-q4f32_1-MLC
    +Mistral-7B-Instruct-v0.2,https://huggingface.co/mistralai/Mistral-7B-Instruct-v0.2,mistral,32768,mistral_default,HF://junrushao/Mistral-7B-Instruct-v0.2-q3f16_1-MLC,HF://junrushao/Mistral-7B-Instruct-v0.2-q4f16_1-MLC,HF://junrushao/Mistral-7B-Instruct-v0.2-q4f32_1-MLC
    +NeuralHermes-2.5-Mistral-7B,https://huggingface.co/mlabonne/NeuralHermes-2.5-Mistral-7B,mistral,32768,neural_hermes_mistral,HF://junrushao/NeuralHermes-2.5-Mistral-7B-q3f16_1-MLC,HF://junrushao/NeuralHermes-2.5-Mistral-7B-q4f16_1-MLC,HF://junrushao/NeuralHermes-2.5-Mistral-7B-q4f32_1-MLC
    +OpenHermes-2-Mistral-7B,https://huggingface.co/teknium/OpenHermes-2-Mistral-7B,mistral,32768,open_hermes_mistral,HF://junrushao/OpenHermes-2-Mistral-7B-q3f16_1-MLC,HF://junrushao/OpenHermes-2-Mistral-7B-q4f16_1-MLC,HF://junrushao/OpenHermes-2-Mistral-7B-q4f32_1-MLC
    +RedPajama-INCITE-Chat-3B-v1,https://huggingface.co/togethercomputer/RedPajama-INCITE-Chat-3B-v1,gpt_neox,2048,redpajama_chat,HF://junrushao/RedPajama-INCITE-Chat-3B-v1-q3f16_1-MLC,HF://junrushao/RedPajama-INCITE-Chat-3B-v1-q4f16_1-MLC,HF://junrushao/RedPajama-INCITE-Chat-3B-v1-q4f32_1-MLC
    +phi-2,https://huggingface.co/microsoft/phi-2,phi-msft,2048,phi-2,HF://junrushao/phi-2-q3f16_1-MLC,HF://junrushao/phi-2-q4f16_1-MLC,HF://junrushao/phi-2-q4f32_1-MLC
    +phi-1_5,https://huggingface.co/microsoft/phi-1_5,phi-msft,2048,phi-2,HF://junrushao/phi-1_5-q3f16_1-MLC,HF://junrushao/phi-1_5-q4f16_1-MLC,HF://junrushao/phi-1_5-q4f32_1-MLC
    +vicuna-7b-v1.5,https://huggingface.co/lmsys/vicuna-7b-v1.5,llama,4096,vicuna_v1.1,HF://junrushao/vicuna-7b-v1.5-q3f16_1-MLC,HF://junrushao/vicuna-7b-v1.5-q4f16_1-MLC,HF://junrushao/vicuna-7b-v1.5-q4f32_1-MLC
    +gpt2,https://huggingface.co/gpt2,gpt2,1024,gpt2,HF://junrushao/gpt2-q3f16_1-MLC,HF://junrushao/gpt2-q4f16_1-MLC,HF://junrushao/gpt2-q4f32_1-MLC
    +gpt2-medium,https://huggingface.co/gpt2-medium,gpt2,1024,gpt2,HF://junrushao/gpt2-medium-q3f16_1-MLC,HF://junrushao/gpt2-medium-q4f16_1-MLC,HF://junrushao/gpt2-medium-q4f32_1-MLC
    +OpenOrca-Platypus2-13B,https://huggingface.co/Open-Orca/OpenOrca-Platypus2-13B,llama,4096,llama-2,HF://junrushao/OpenOrca-Platypus2-13B-q3f16_1-MLC,HF://junrushao/OpenOrca-Platypus2-13B-q4f16_1-MLC,HF://junrushao/OpenOrca-Platypus2-13B-q4f32_1-MLC
    +Llama2-Chinese-7b-Chat,https://huggingface.co/FlagAlpha/Llama2-Chinese-7b-Chat,llama,4096,llama-2,HF://junrushao/Llama2-Chinese-7b-Chat-q3f16_1-MLC,HF://junrushao/Llama2-Chinese-7b-Chat-q4f16_1-MLC,HF://junrushao/Llama2-Chinese-7b-Chat-q4f32_1-MLC
    +Llama2-Chinese-13b-Chat,https://huggingface.co/FlagAlpha/Llama2-Chinese-13b-Chat,llama,4096,llama-2,HF://junrushao/Llama2-Chinese-13b-Chat-q3f16_1-MLC,HF://junrushao/Llama2-Chinese-13b-Chat-q4f16_1-MLC,HF://junrushao/Llama2-Chinese-13b-Chat-q4f32_1-MLC
    +YuLan-Chat-2-13b-fp16,https://huggingface.co/yulan-team/YuLan-Chat-2-13b-fp16,llama,8192,llama-2,HF://junrushao/YuLan-Chat-2-13b-fp16-q3f16_1-MLC,HF://junrushao/YuLan-Chat-2-13b-fp16-q4f16_1-MLC,HF://junrushao/YuLan-Chat-2-13b-fp16-q4f32_1-MLC
    \ No newline at end of file
    diff --git a/wasted/test000.py b/wasted/test000.py
    new file mode 100644
    index 0000000000..4333be36ad
    --- /dev/null
    +++ b/wasted/test000.py
    @@ -0,0 +1,54 @@
    +# pylint: skip-file
    +import argparse
    +
    +import numpy as np
    +import torch
    +import tvm
    +from transformers import AutoTokenizer
    +from tvm import relax
    +from tvm.relax.frontend.nn import IOEffect
    +from tvm.runtime import disco as di
    +
    +import mlc_chat
    +
    +
    +def deploy_to_pipeline(args) -> None:
    +    devices = list(range(args.tensor_parallel_shards))
    +    print(devices)
    +    sess = di.ProcessSession(
    +        num_workers=args.tensor_parallel_shards, entrypoint="mlc_chat.cli.worker"
    +    )
    +    sess.init_ccl("nccl", *devices)
    +    print(args.model_lib)
    +    mod = sess.load_vm_module(args.model_lib)
    +    create_kv_cache = mod["create_kv_cache"]
    +
    +    loader = sess.get_global_func("mlc.loader.LoadMultiGPU")
    +    params = loader(args.params, mod)
    +    exit(0)
    +
    +
    +def _parse_args():
    +    args = argparse.ArgumentParser()
    +    args.add_argument("--model-lib", type=str, required=True)
    +    args.add_argument("--params", type=str, required=True)
    +    args.add_argument("--tensor-parallel-shards", type=int, default=4)
    +    args.add_argument("--primary-device", type=str, default="auto")
    +    args.add_argument("--prompt", type=str, default="What is the meaning of life?")
    +    args.add_argument("--time-eval", default=False, action="store_true")
    +    args.add_argument("--skip-rounds", type=int, default=0)
    +    parsed = args.parse_args()
    +
    +    if parsed.primary_device == "auto":
    +        if tvm.cuda().exist:
    +            parsed.primary_device = "cuda"
    +        elif tvm.metal().exist:
    +            parsed.primary_device = "metal"
    +        else:
    +            raise ValueError("Cannot auto deduce device-name, please set it")
    +    return parsed
    +
    +
    +if __name__ == "__main__":
    +    args = _parse_args()
    +    deploy_to_pipeline(args)
    diff --git a/wasted/test_7mistral.sh b/wasted/test_7mistral.sh
    new file mode 100755
    index 0000000000..648abb5e11
    --- /dev/null
    +++ b/wasted/test_7mistral.sh
    @@ -0,0 +1,11 @@
    +NUM_SHARDS=2
    +MODEL_PATH=/opt/models/llama-2/llama-2-7b-chat-hf/
    +OUTPUT_PATH=./dist/Mixtral-8x7B-Instruct-v0.1-q0f16/params
    +
    +# python -m mlc_chat gen_config --model $MODEL_PATH/config.json --quantization q4f16_1 --conv-template llama-2 --context-window-size 4096 --num-shards $NUM_SHARDS -o $OUTPUT_PATH
    +
    +# python -m mlc_chat compile --model $OUTPUT_PATH -o $OUTPUT_PATH/llama.so
    +
    +# python -m mlc_chat convert_weight --model $MODEL_PATH --quantization q4f16_1 -o $OUTPUT_PATH
    +
    +CUDA_VISIBLE_DEVICES=6,7 python -m mlc_chat.cli.benchmark --model ./dist/Mixtral-8x7B-Instruct-v0.1-q0f16/params --model-lib ./dist/Mixtral-8x7B-Instruct-v0.1-q0f16/Mixtral-8x7B-Instruct-v0.1-q0f16-cuda.so --device "cuda:0" --prompt "What is the meaning of life?" --generate-length 256 --num-shards 2
    diff --git a/wasted/test_awq.sh b/wasted/test_awq.sh
    new file mode 100755
    index 0000000000..9742bb5151
    --- /dev/null
    +++ b/wasted/test_awq.sh
    @@ -0,0 +1,10 @@
    +MODEL_PATH=/opt/models/llama-2/llama-2-7b-chat-hf/
    +OUTPUT_PATH=./dist/new-llama-awq-mit/
    +
    +# python -m mlc_chat compile --model llama2_7b --quantization q4f16_awq -o $OUTPUT_PATH/llama.so --context-window-size 4096
    +
    +python -m mlc_chat convert_weight $MODEL_PATH --quantization q4f16_awq -o $OUTPUT_PATH --source-format awq --source dist/models/llama-2-7b-chat-w4-g128-awq.pt
    +
    +# python -m mlc_chat gen_mlc_chat_config --model $MODEL_PATH/config.json --quantization q4f16_awq -o $OUTPUT_PATH --conv-template llama-2
    +
    +# CUDA_VISIBLE_DEVICES=4,5,6,7 python -m mlc_chat.cli.benchmark --model $OUTPUT_PATH --model-lib $OUTPUT_PATH/llama.so --device "cuda:0" --prompt "What is the meaning of life?" --generate-length 256
    diff --git a/wasted/test_awq_gemm.sh b/wasted/test_awq_gemm.sh
    new file mode 100755
    index 0000000000..75176becf5
    --- /dev/null
    +++ b/wasted/test_awq_gemm.sh
    @@ -0,0 +1,14 @@
    +#!/bin/bash
    +set -e
    +
    +MODEL_PATH=/opt/models/llama-2/llama-2-7b-chat-hf/
    +OUTPUT_PATH=./dist/new-llama-awq/
    +QUANT=q4f16_autoawq
    +
    +# python -m mlc_chat gen_config $MODEL_PATH/config.json --quantization $QUANT -o $OUTPUT_PATH --conv-template llama-2
    +
    +# python -m mlc_chat compile $OUTPUT_PATH -o $OUTPUT_PATH/llama.so
    +
    +python -m mlc_chat convert_weight $MODEL_PATH --quantization $QUANT -o $OUTPUT_PATH --source-format awq --source ../Llama-2-7B-Chat-AWQ/model.safetensors
    +
    +CUDA_VISIBLE_DEVICES=4,5,6,7 python -m mlc_chat.cli.benchmark --model $OUTPUT_PATH --model-lib $OUTPUT_PATH/llama.so --device "cuda:0" --prompt "What is the meaning of life?" --generate-length 256
    diff --git a/wasted/test_decode_awq_gemm.py b/wasted/test_decode_awq_gemm.py
    new file mode 100644
    index 0000000000..0fdd872694
    --- /dev/null
    +++ b/wasted/test_decode_awq_gemm.py
    @@ -0,0 +1,110 @@
    +# pylint: disable=invalid-name,missing-docstring
    +from typing import List
    +
    +import numpy as np
    +import pytest
    +import safetensors
    +import torch
    +import tvm
    +import tvm.testing
    +from tvm import DataType
    +from tvm.relax.frontend import nn
    +
    +from mlc_chat.compiler import QUANTIZATION
    +from mlc_chat.compiler.loader import QuantizeMapping
    +from mlc_chat.compiler.quantization import AWQQuantize
    +from mlc_chat.support.auto_device import detect_device
    +from mlc_chat.support.auto_target import detect_target_and_host
    +
    +
    +def dequantize_np(
    +    config: AWQQuantize,
    +    weight: np.ndarray,
    +    zeros: np.ndarray,
    +    scale: np.ndarray,
    +) -> np.ndarray:
    +    def decode_int_arr(int_arr: np.ndarray, num_elem_per_storage: int, bits: int):
    +        bin_mask = (1 << bits) - 1
    +        int_arr_repeated = np.repeat(int_arr, num_elem_per_storage, axis=-1)
    +        indice_j = np.indices(int_arr_repeated.shape)[1]
    +        inv_order_map = [0, 4, 1, 5, 2, 6, 3, 7]
    +        arr_bin = np.bitwise_and(
    +            np.right_shift(
    +                int_arr_repeated,
    +                inv_order_map[indice_j % num_elem_per_storage] * bits,
    +            ),
    +            bin_mask,
    +        )
    +        return arr_bin
    +
    +    weight_bin = decode_int_arr(
    +        weight, config.num_elem_per_storage, DataType(config.quantize_dtype).bits
    +    )
    +    zero_bin = decode_int_arr(
    +        zeros, config.num_elem_per_storage, DataType(config.quantize_dtype).bits
    +    )
    +    scale_repeated = np.repeat(scale, config.group_size, axis=-1)
    +    zero_bin_repeated = np.repeat(zero_bin, config.group_size, axis=-1)
    +    return (weight_bin - zero_bin_repeated) * scale_repeated
    +
    +
    +def test_dequantize_weight(quant_name: str, dtype: str):
    +    target, _ = detect_target_and_host("auto")
    +    device = detect_device("auto")
    +    # print(target)
    +    # print(device)
    +    with safetensors.safe_open(
    +        "../Llama-2-7B-AWQ/model.safetensors", framework="numpy", device="cpu"
    +    ) as in_file:
    +        q_proj_qweight = in_file.get_tensor("model.layers.0.self_attn.q_proj.qweight").astype(
    +            "uint32"
    +        )
    +        q_proj_qzeros = in_file.get_tensor("model.layers.0.self_attn.q_proj.qzeros").astype(
    +            "uint32"
    +        )
    +        q_proj_scales = in_file.get_tensor("model.layers.0.self_attn.q_proj.scales")
    +    hf_state_dict = torch.load("/opt/models/llama-2/llama-2-7b-hf/pytorch_model-00001-of-00002.bin")
    +    q_proj_weight = hf_state_dict["model.layers.0.self_attn.q_proj.weight"]
    +    print(q_proj_weight)
    +
    +    class Test(nn.Module):
    +        def __init__(self) -> None:
    +            super().__init__()
    +            self.linear = nn.Linear(
    +                q_proj_weight.shape[1], q_proj_weight.shape[0], bias=False, dtype=dtype
    +            )
    +
    +        def forward(self, x: nn.Tensor):
    +            return self.linear(x)
    +
    +    config = QUANTIZATION[quant_name]
    +    assert isinstance(config, AWQQuantize)
    +    mod = config.quantize_model(Test(), QuantizeMapping({}, {}), "")
    +    mod.linear.qweight.data = q_proj_qweight
    +    mod.linear.qzeros.data = q_proj_qzeros
    +    mod.linear.scales.data = q_proj_scales
    +    with target:
    +        model = mod.jit(
    +            spec={
    +                "forward": {
    +                    "x": nn.spec.Tensor((q_proj_weight.shape[1], q_proj_weight.shape[1]), dtype)
    +                }
    +            },
    +            pipeline="mlc_llm",
    +            device="cuda:0",
    +            target=target,
    +        )
    +    out = model["forward"](
    +        torch.from_numpy(np.diag(np.ones(q_proj_weight.shape[1]).astype(dtype))).to(
    +            "cuda:0"
    +        )  # pylint: disable=no-member
    +    )
    +    print(out)
    +    print(q_proj_weight.shape)
    +    print(out.shape)
    +    # out = dequantize_np(config, q_proj_qweight, q_proj_qzeros, q_proj_scales)
    +    print(torch.max(torch.abs(out.detach().to("cpu") - q_proj_weight)))
    +
    +
    +if __name__ == "__main__":
    +    test_dequantize_weight("q4f16_awq_gemm", "float16")
    diff --git a/wasted/test_mistral.py b/wasted/test_mistral.py
    new file mode 100644
    index 0000000000..895de58d1c
    --- /dev/null
    +++ b/wasted/test_mistral.py
    @@ -0,0 +1,54 @@
    +# pylint: skip-file
    +import os
    +import shutil
    +
    +import tvm
    +
    +tvm.error.InternalError
    +
    +temp_dir = "/opt/scratch/lesheng/mlc-llm/dist/tmp"
    +
    +os.environ["MLC_TEMP_DIR"] = temp_dir
    +os.environ["MLC_CACHE_DIR"] = temp_dir
    +
    +import logging
    +
    +from mlc_chat import ChatConfig, ChatModule
    +from mlc_chat.callback import StreamToStdout
    +
    +logging.basicConfig(
    +    level=logging.INFO,
    +    style="{",
    +    datefmt="%Y-%m-%d %H:%M:%S",
    +    format="[{asctime}] {levelname} {filename}:{lineno}: {message}",
    +)
    +
    +import json
    +
    +
    +def test_model(url):
    +    os.makedirs(temp_dir, exist_ok=True)
    +    cm = ChatModule(
    +        url,
    +        chat_config=ChatConfig(max_gen_len=10000, prefill_chunk_size=4096),
    +        device="cuda",
    +    )
    +    output = cm.generate(
    +        prompt="What is the meaning of life?",
    +    )
    +    print(output)
    +    print(cm.stats())
    +    for i in range(10):
    +        prompt = input("User: ")
    +        output = cm.generate(
    +            prompt=prompt,
    +        )
    +        print(output)
    +        print(cm.stats())
    +    return output
    +
    +
    +url = "HF://junrushao/Mistral-7B-Instruct-v0.2-q4f16_1-MLC"
    +# url = "HF://junrushao/phi-2-q4f16_1-MLC"
    +# url = "HF://junrushao/Llama-2-7b-chat-hf-q4f16_1-MLC"
    +test_model(url)
    diff --git a/wasted/test_mistral_multi_gpu.sh b/wasted/test_mistral_multi_gpu.sh
    new file mode 100644
    index 0000000000..440d0f6caf
    --- /dev/null
    +++ b/wasted/test_mistral_multi_gpu.sh
    @@ -0,0 +1,19 @@
    +QUANT=q4f16_1
    +MODEL_PATH=/opt/models/llama-2/llama-2-7b-chat-hf
    +OUTPUT_PATH=./dist/new-llama-${QUANT}/
    +
    +mkdir -p $OUTPUT_PATH
    +
    +python -m mlc_chat gen_config $MODEL_PATH/config.json --quantization $QUANT --conv-template llama-2 --num-shards $NUM_SHARDS -o $OUTPUT_PATH/params
    +
    +# python -m mlc_chat compile $OUTPUT_PATH/params -o $OUTPUT_PATH/llama.so
    +
    +# python -m mlc_chat convert_weight $MODEL_PATH --quantization $QUANT -o $OUTPUT_PATH/params
    +
    +CUDA_VISIBLE_DEVICES=4,5,6,7 python -m mlc_chat.cli.benchmark --model $OUTPUT_PATH/params --model-lib $OUTPUT_PATH/llama.so --device "cuda:0" --prompt "What is the meaning of life?" --generate-length 256 --num-shards 1
    +
    +# python -m mlc_chat compile --model $OUTPUT_PATH -o $OUTPUT_PATH/llama.so
    +
    +# python -m mlc_chat convert_weight --model $MODEL_PATH --quantization q4f16_1 -o $OUTPUT_PATH
    +
    +CUDA_VISIBLE_DEVICES=4,5,6,7 python -m mlc_chat.cli.benchmark --model $OUTPUT_PATH --model-lib $OUTPUT_PATH/llama.so --device "cuda:0" --prompt "What is the meaning of life?" --generate-length 256
    diff --git a/wasted/test_mlc.py b/wasted/test_mlc.py
    new file mode 100644
    index 0000000000..38a5f812dd
    --- /dev/null
    +++ b/wasted/test_mlc.py
    @@ -0,0 +1,30 @@
    +import tvm
    +
    +from mlc_chat import ChatModule
    +from mlc_chat.chat_module import ChatConfig
    +
    +
    +@tvm.register_func("effect.print")
    +def _print(_, array) -> None:
    +    print(f"effect.print: shape = {array.shape}, dtype = {array.dtype}, data =\n{array}")
    +
    +
    +import numpy as np
    +
    +array = tvm.nd.array(np.random.random((4, 20)))
    +
    +print(f"effect.print: shape = {array.shape}, dtype = {array.dtype}, data =\n{array}")
    +
    +# cm = ChatModule(
    +#     # model="dist/llama-2-7b-chat-hf-q4f16_1/params",
    +#     model="dist/new-llama-q4f16_1/params",
    +#     chat_config=ChatConfig(
    +#         temperature=0.01,
    +#         top_p=0.9,
    +#         max_gen_len=4096,
    +#     ),
    +#     model_lib_path="dist/new-llama-q4f16_1/llama.so",
    +# )
    +
    +# output = cm.generate("What is the meaning of life?")
    +# print(output)
    diff --git a/wasted/test_mlc_hf.py b/wasted/test_mlc_hf.py
    new file mode 100644
    index 0000000000..e608048934
    --- /dev/null
    +++ b/wasted/test_mlc_hf.py
    @@ -0,0 +1,23 @@
    +import os
    +
    +os.environ["MLC_TEMP_DIR"] = "/opt/scratch/lesheng/mlc-llm/dist"
    +os.environ["MLC_CACHE_DIR"] = "/opt/scratch/lesheng/mlc-llm/dist"
    +
    +import logging
    +
    +from mlc_chat import ChatModule
    +from mlc_chat.callback import StreamToStdout
    +
    +logging.basicConfig(
    +    level=logging.INFO,
    +    style="{",
    +    datefmt="%Y-%m-%d %H:%M:%S",
    +    format="[{asctime}] {levelname} {filename}:{lineno}: {message}",
    +)
    +
    +cm = ChatModule("HF://junrushao/Llama-2-7b-chat-hf-q3f16_1-MLC", device="cuda")
    +output = cm.generate(
    +    prompt="What is the meaning of life?",
    +    # progress_callback=StreamToStdout(callback_interval=2),
    +)
    +print(output)
    diff --git a/wasted/test_mlc_hf_batch.py b/wasted/test_mlc_hf_batch.py
    new file mode 100644
    index 0000000000..081f328153
    --- /dev/null
    +++ b/wasted/test_mlc_hf_batch.py
    @@ -0,0 +1,81 @@
    +# pylint: skip-file
    +import os
    +import shutil
    +
    +import tvm
    +
    +tvm.error.InternalError
    +
    +temp_dir = "/opt/scratch/lesheng/mlc-llm/dist/tmp"
    +
    +os.environ["MLC_TEMP_DIR"] = temp_dir
    +os.environ["MLC_CACHE_DIR"] = temp_dir
    +
    +import logging
    +
    +from mlc_chat import ChatConfig, ChatModule
    +from mlc_chat.callback import StreamToStdout
    +
    +logging.basicConfig(
    +    level=logging.INFO,
    +    style="{",
    +    datefmt="%Y-%m-%d %H:%M:%S",
    +    format="[{asctime}] {levelname} {filename}:{lineno}: {message}",
    +)
    +
    +import json
    +
    +
    +def test_model(url):
    +    os.makedirs(temp_dir, exist_ok=True)
    +    try:
    +        cm = ChatModule(
    +            url,
    +            chat_config=ChatConfig(max_gen_len=10000, prefill_chunk_size=4096),
    +            device="cuda",
    +        )
    +        output = cm.generate(
    +            prompt="What is the meaning of life?",
    +            # prompt="Write a python program that computes fibonacci sequence."
    +        )
    +        print(output)
    +        for i in range(10):
    +            output = cm.generate(
    +                prompt="Talk more",
    +                # prompt="Write a python program that computes fibonacci sequence."
    +            )
    +            print(output)
    +    except:
    +        output = "Failed"
    +    # shutil.rmtree(temp_dir, ignore_errors=True)
    +    return output
    +
    +
    +url = "HF://junrushao/Mistral-7B-Instruct-v0.2-q4f16_1-MLC"
    +output = test_model(url)
    +print(output)
    +exit(0)
    +
    +with open("model_info.json", "r") as f:
    +    model_dict = json.load(f)
    +
    +testout_dir = "testout"
    +os.makedirs(testout_dir, exist_ok=True)
    +
    +shutil.rmtree(temp_dir, ignore_errors=True)
    +for model in model_dict:
    +    model_infos = model_dict[model]
    +    for quant in ["q3f16_1", "q4f16_1", "q4f32_1"]:
    +        url = model_infos[quant]
    +        if len(url) == 0:
    +            continue
    +        print(f"{model}\t{quant}: {url}")
    +        testout_file = os.path.join(testout_dir, f"{model}_{quant}.out")
    +        if f"{model}_{quant}" in ["CodeLlama-7b-hf_q4f32_1"]:
    +            continue
    +        if os.path.exists(testout_file):
    +            print(f"skip {testout_file}")
    +            continue
    +        output = test_model(url)
    +        with open(testout_file, "w+") as f:
    +            f.write(output + "\n")
    diff --git a/wasted/test_mlc_phi.py b/wasted/test_mlc_phi.py
    new file mode 100644
    index 0000000000..ed8a54b60f
    --- /dev/null
    +++ b/wasted/test_mlc_phi.py
    @@ -0,0 +1,16 @@
    +from mlc_chat import ChatModule
    +from mlc_chat.chat_module import ChatConfig
    +
    +cm = ChatModule(
    +    model="./dist/phi-2-q0f16/params",
    +    model_lib_path="./dist/phi-2-q0f16/model.so",
    +    # chat_config=ChatConfig(
    +    #     temperature=0.01,
    +    #     top_p=0.9,
    +    #     max_gen_len=4096,
    +    # ),
    +)
    +
    +output = cm.generate("What is the meaning of life?")
    +print(output)
    +print(output)
    diff --git a/wasted/test_mlc_red_pajama.py b/wasted/test_mlc_red_pajama.py
    new file mode 100644
    index 0000000000..43b708e7bb
    --- /dev/null
    +++ b/wasted/test_mlc_red_pajama.py
    @@ -0,0 +1,14 @@
    +from mlc_chat import ChatModule
    +from mlc_chat.chat_module import ChatConfig
    +
    +cm = ChatModule(
    +    model="dist/RedPajama-INCITE-Chat-3B-v1-q0f16/params",
    +    chat_config=ChatConfig(
    +        temperature=0.01,
    +        top_p=0.9,
    +        max_gen_len=4096,
    +    ),
    +)
    +
    +output = cm.generate("What is the meaning of life?")
    +print(output)
    diff --git a/wasted/test_old_red_pajama.sh b/wasted/test_old_red_pajama.sh
    new file mode 100755
    index 0000000000..63ce6a50db
    --- /dev/null
    +++ b/wasted/test_old_red_pajama.sh
    @@ -0,0 +1,3 @@
    +MODEL_PATH=/opt/scratch/lesheng/models/RedPajama-INCITE-Chat-3B-v1
    +
    +CUDA_VISIBLE_DEVICES=4,5,6,7 python build.py --model $MODEL_PATH --quantization q4f16_1 --use-cache 0 --debug-dump # --build-model-only
    diff --git a/wasted/test_phi_hf.py b/wasted/test_phi_hf.py
    new file mode 100644
    index 0000000000..c738776e43
    --- /dev/null
    +++ b/wasted/test_phi_hf.py
    @@ -0,0 +1,19 @@
    +import torch
    +from transformers import AutoModelForCausalLM, AutoTokenizer
    +
    +torch.set_default_device("cuda")
    +
    +model = AutoModelForCausalLM.from_pretrained(
    +    "/opt/scratch/lesheng/phi-2", torch_dtype="auto", trust_remote_code=True
    +)
    +tokenizer = AutoTokenizer.from_pretrained("/opt/scratch/lesheng/phi-2", trust_remote_code=True)
    +
    +inputs = tokenizer(
    +    "What is the meaning of life?",
    +    return_tensors="pt",
    +    return_attention_mask=False,
    +)
    +
    +outputs = model.generate(**inputs, max_length=200)
    +text = tokenizer.batch_decode(outputs)[0]
    +print(text)
    diff --git a/wasted/test_phi_mlc.py b/wasted/test_phi_mlc.py
    new file mode 100644
    index 0000000000..482764cd26
    --- /dev/null
    +++ b/wasted/test_phi_mlc.py
    @@ -0,0 +1,67 @@
    +import torch
    +from transformers import AutoModelForCausalLM, AutoTokenizer
    +from tvm.relax.frontend import nn
    +
    +from mlc_chat.compiler.model.phi import modeling_phi as hf_phi
    +from mlc_chat.compiler.model.phi import phi_model
    +
    +config = phi_model.PhiConfig()
    +
    +
    +def test_rotary():
    +    rotary_embed = phi_model.RotaryEmbedding(config)
    +    print(config.n_head)
    +    print(config.head_dim)
    +    print(rotary_embed.position_embedding_base)
    +    print(rotary_embed.rotary_dim)
    +    mod = rotary_embed.jit(
    +        spec={
    +            "forward": {
    +                "q": nn.spec.Tensor([1, 7, config.n_head, config.head_dim], "float16"),
    +                "k": nn.spec.Tensor([1, 7, config.n_head, config.head_dim], "float16"),
    +                "offset": int,
    +            }
    +        }
    +    )
    +    q = torch.rand(1, 7, config.n_head, config.head_dim, dtype=torch.float16)
    +    k = torch.rand(1, 7, config.n_head, config.head_dim, dtype=torch.float16)
    +    q_embed, k_embed = mod["forward"](q, k, 0)
    +
    +    hf_rotary = hf_phi.RotaryEmbedding(config.rotary_dim)
    +    kv = torch.stack([k, k], dim=2)
    +    hf_q, hf_kv = hf_rotary.forward(q, kv, 0)
    +    print(torch.allclose(hf_q, q_embed))
    +    # print(hf_q)
    +    # print(q_embed)
    +    hf_k, _ = torch.unbind(hf_kv, 2)
    +    print(torch.allclose(hf_k, k_embed))
    +    # print(q_embed.shape)
    +
    +
    +def test_e2e():
    +    hf_model = AutoModelForCausalLM.from_pretrained(
    +        "/opt/scratch/lesheng/phi-2", torch_dtype=torch.float16, trust_remote_code=True
    +    )
    +    tokenizer = AutoTokenizer.from_pretrained("/opt/scratch/lesheng/phi-2", trust_remote_code=True)
    +
    +    state_dict = hf_model.state_dict()
    +    state_dict["transformer.embd.weight"] = state_dict["transformer.embd.wte.weight"]
    +    state_dict.pop("transformer.embd.wte.weight")
    +    phi = phi_model.PhiForCausalLM(config)
    +    phi.to("float16")
    +    phi.load_state_dict(state_dict)
    +    mod = phi.jit(spec=phi.get_default_spec())
    +
    +    inputs = tokenizer(
    +        "What is the meaning of life?",
    +        return_tensors="pt",
    +        return_attention_mask=False,
    +    )
    +
    +    mod["prefill"](inputs, len(inputs[0]))
    +
    +
    +if __name__ == "__main__":
    +    test_rotary()
    +    # test_e2e()
    +    # test_e2e()
    diff --git a/wasted/test_preshard.sh b/wasted/test_preshard.sh
    new file mode 100755
    index 0000000000..504a80c719
    --- /dev/null
    +++ b/wasted/test_preshard.sh
    @@ -0,0 +1,7 @@
    +set -e
    +
    +MODEL_PATH=/opt/models/llama-2/llama-2-7b-chat-hf/
    +
    +CUDA_VISIBLE_DEVICES=4,5,6,7 python build.py --model $MODEL_PATH --quantization q4f16_1 --num-shards 2 --build-model-only --max-seq-len 4096 --use-cache 0 --use-presharded-weights
    +
    +# CUDA_VISIBLE_DEVICES=4,5,6,7 python build.py --model $MODEL_PATH --quantization q4f16_1 --num-shards 2 --convert-weight-only --max-seq-len 4096 --use-cache 0 --use-presharded-weights
    diff --git a/wasted/worker.py b/wasted/worker.py
    new file mode 100644
    index 0000000000..59e2179016
    --- /dev/null
    +++ b/wasted/worker.py
    @@ -0,0 +1,64 @@
    +# Licensed to the Apache Software Foundation (ASF) under one
    +# or more contributor license agreements.  See the NOTICE file
    +# distributed with this work for additional information
    +# regarding copyright ownership.  The ASF licenses this file
    +# to you under the Apache License, Version 2.0 (the
    +# "License"); you may not use this file except in compliance
    +# with the License.  You may obtain a copy of the License at
    +#
    +#   http://www.apache.org/licenses/LICENSE-2.0
    +#
    +# Unless required by applicable law or agreed to in writing,
    +# software distributed under the License is distributed on an
    +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
    +# KIND, either express or implied.  See the License for the
    +# specific language governing permissions and limitations
    +# under the License.
    +# pylint: disable=invalid-name
    +"""Internal DiscoWorker for Disco ProcessSession."""
    +import os
    +import sys
    +
    +import numpy as np
    +from tvm import register_func
    +from tvm import runtime as _  # pylint: disable=unused-import
    +from tvm._ffi import get_global_func
    +
    +from .. import base  # pylint: disable=unused-import, no-name-in-module
    +
    +
    +def main():
    +    """Main worker function"""
    +    if len(sys.argv) != 5:
    +        print("Usage:    ")
    +        return
    +
    +    worker_id = int(sys.argv[1])
    +    num_workers = int(sys.argv[2])
    +    if sys.platform == "win32":
    +        import msvcrt  # pylint: disable=import-outside-toplevel,import-error
    +
    +        reader = msvcrt.open_osfhandle(int(sys.argv[3]), os.O_BINARY)
    +        writer = msvcrt.open_osfhandle(int(sys.argv[4]), os.O_BINARY)
    +    else:
    +        reader = int(sys.argv[3])
    +        writer = int(sys.argv[4])
    +
    +    @register_func("vm.builtin.debug_print_disco")
    +    def _print(lineo: str, array) -> None:
    +        np_array = array.numpy()
    +        np.save(f"disco_test/{worker_id}.npy", np_array)
    +        print(f"{worker_id} done")
    +        # print(
    +        #     f"{worker_id}, {lineo}: shape = {array.shape}, dtype = {array.dtype}, data =\n{array}"
    +        # )
    +
    +    worker_func = get_global_func("runtime.disco.WorkerProcess")
    +    worker_func(worker_id, num_workers, reader, writer)
    +
    +
    +if __name__ == "__main__":
    +    try:
    +        main()
    +    except (KeyboardInterrupt, IOError):
    +        pass