fix

README.md

@@ -1,7 +1,7 @@
 ---
 license: other
 license_name: glm-4
-license_link: https://huggingface.co/THUDM/glm-4-9b/LICENSE
+license_link: https://huggingface.co/THUDM/glm-4-9b/main/LICENSE
 language:
 - zh
 - en

config.json

@@ -36,7 +36,7 @@
  "seq_length": 8192,
  "use_cache": true,
  "torch_dtype": "bfloat16",
- "transformers_version": "4.40.2",
+ "transformers_version": "4.42.4",
  "tie_word_embeddings": false,
  "eos_token_id": [151329, 151336, 151338],
  "pad_token_id": 151329

generation_config.json

@@ -9,5 +9,5 @@
  "temperature": 0.8,
  "max_length": 8192,
  "top_p": 0.8,
- "transformers_version": "4.40.2"
+ "transformers_version": "4.42.4"
 }

modeling_chatglm.py

@@ -797,11 +797,6 @@ class ChatGLMPreTrainedModel(PreTrainedModel):
  position_ids = torch.arange(seq_length, dtype=torch.long, device=device).unsqueeze(0).repeat(batch_size, 1)
  return position_ids
 
- def gradient_checkpointing_enable(self, gradient_checkpointing_kwargs=None):
- if not self.supports_gradient_checkpointing:
- raise ValueError(f"{self.__class__.__name__} does not support gradient checkpointing.")
-
-
 class Embedding(torch.nn.Module):
  """Language model embeddings."""
 
@@ -936,9 +931,10 @@ class ChatGLMForConditionalGeneration(ChatGLMPreTrainedModel):
  standardize_cache_format: bool = False,
  ) -> Dict[str, Any]:
  # update past_key_values
- model_kwargs["past_key_values"] = self._extract_past_from_model_output(
+ cache_name, cache = self._extract_past_from_model_output(
  outputs, standardize_cache_format=standardize_cache_format
  )
+ model_kwargs[cache_name] = cache
 
  # update attention mask
  if "attention_mask" in model_kwargs:
@@ -1063,202 +1059,6 @@ class ChatGLMForConditionalGeneration(ChatGLMPreTrainedModel):
  for layer_past in past
  )
 
- def process_response(self, output, history):
- content = ""
- history = deepcopy(history)
- for response in output.split("<|assistant|>"):
- if "\n" in response:
- metadata, content = response.split("\n", maxsplit=1)
- else:
- metadata, content = "", response
- if not metadata.strip():
- content = content.strip()
- history.append({"role": "assistant", "metadata": metadata, "content": content})
- content = content.replace("[[训练时间]]", "2023年")
- else:
- history.append({"role": "assistant", "metadata": metadata, "content": content})
- if history[0]["role"] == "system" and "tools" in history[0]:
- parameters = json.loads(content)
- content = {"name": metadata.strip(), "parameters": parameters}
- else:
- content = {"name": metadata.strip(), "content": content}
- return content, history
-
- @torch.inference_mode()
- def chat(self, tokenizer, query: str, history: List[Dict] = None, role: str = "user",
- max_length: int = 8192, num_beams=1, do_sample=True, top_p=0.8, temperature=0.8, logits_processor=None,
- **kwargs):
- if history is None:
- history = []
- if logits_processor is None:
- logits_processor = LogitsProcessorList()
- logits_processor.append(InvalidScoreLogitsProcessor())
- gen_kwargs = {"max_length": max_length, "num_beams": num_beams, "do_sample": do_sample, "top_p": top_p,
- "temperature": temperature, "logits_processor": logits_processor, **kwargs}
- history.append({"role": role, "content": query})
- inputs = tokenizer.apply_chat_template(history, add_generation_prompt=True, tokenize=True,
- return_tensors="pt", return_dict=True)
- inputs = inputs.to(self.device)
- eos_token_id = [tokenizer.eos_token_id, tokenizer.convert_tokens_to_ids("<|user|>"),
- tokenizer.convert_tokens_to_ids("<|observation|>")]
- outputs = self.generate(**inputs, **gen_kwargs, eos_token_id=eos_token_id)
- outputs = outputs.tolist()[0][len(inputs["input_ids"][0]):-1]
- response = tokenizer.decode(outputs)
- response, history = self.process_response(response, history)
- return response, history
-
- @torch.inference_mode()
- def stream_chat(self, tokenizer, query: str, history: List[Dict] = None, role: str = "user",
- past_key_values=None, max_length: int = 8192, do_sample=True, top_p=0.8, temperature=0.8,
- logits_processor=None, return_past_key_values=False, **kwargs):
- if history is None:
- history = []
- if logits_processor is None:
- logits_processor = LogitsProcessorList()
- logits_processor.append(InvalidScoreLogitsProcessor())
- eos_token_id = [tokenizer.eos_token_id, tokenizer.convert_tokens_to_ids("<|user|>"),
- tokenizer.convert_tokens_to_ids("<|observation|>")]
- gen_kwargs = {"max_length": max_length, "do_sample": do_sample, "top_p": top_p,
- "temperature": temperature, "logits_processor": logits_processor, **kwargs}
- if past_key_values is None:
- inputs = tokenizer.apply_chat_template(history + [{"role": role, "content": query}],
- add_generation_prompt=True, tokenize=True, return_tensors="pt",
- return_dict=True)
- else:
- inputs = tokenizer.apply_chat_template([{"role": role, "content": query}], add_special_tokens=False,
- add_generation_prompt=True, tokenize=True, return_tensors="pt",
- return_dict=True)
- inputs = inputs.to(self.device)
- if past_key_values is not None:
- past_length = past_key_values[0][0].shape[2]
- inputs.position_ids += past_length
- attention_mask = inputs.attention_mask
- attention_mask = torch.cat((attention_mask.new_ones(1, past_length), attention_mask), dim=1)
- inputs['attention_mask'] = attention_mask
- history.append({"role": role, "content": query})
- for outputs in self.stream_generate(**inputs, past_key_values=past_key_values,
- eos_token_id=eos_token_id, return_past_key_values=return_past_key_values,
- **gen_kwargs):
- if return_past_key_values:
- outputs, past_key_values = outputs
- outputs = outputs.tolist()[0][len(inputs["input_ids"][0]):-1]
- response = tokenizer.decode(outputs)
- if response and response[-1] != "�":
- response, new_history = self.process_response(response, history)
- if return_past_key_values:
- yield response, new_history, past_key_values
- else:
- yield response, new_history
-
- @torch.inference_mode()
- def stream_generate(
- self,
- input_ids,
- generation_config: Optional[GenerationConfig] = None,
- logits_processor: Optional[LogitsProcessorList] = None,
- stopping_criteria: Optional[StoppingCriteriaList] = None,
- prefix_allowed_tokens_fn: Optional[Callable[[int, torch.Tensor], List[int]]] = None,
- return_past_key_values=False,
- **kwargs,
- ):
- batch_size, input_ids_seq_length = input_ids.shape[0], input_ids.shape[-1]
-
- if generation_config is None:
- generation_config = self.generation_config
- generation_config = copy.deepcopy(generation_config)
- model_kwargs = generation_config.update(**kwargs)
- model_kwargs["use_cache"] = generation_config.use_cache
- bos_token_id, eos_token_id = generation_config.bos_token_id, generation_config.eos_token_id
-
- if isinstance(eos_token_id, int):
- eos_token_id = [eos_token_id]
- eos_token_id_tensor = torch.tensor(eos_token_id).to(input_ids.device) if eos_token_id is not None else None
-
- has_default_max_length = kwargs.get("max_length") is None and generation_config.max_length is not None
- if has_default_max_length and generation_config.max_new_tokens is None:
- warnings.warn(
- f"Using `max_length`'s default ({generation_config.max_length}) to control the generation length. "
- "This behaviour is deprecated and will be removed from the config in v5 of Transformers -- we"
- " recommend using `max_new_tokens` to control the maximum length of the generation.",
- UserWarning,
- )
- elif generation_config.max_new_tokens is not None:
- generation_config.max_length = generation_config.max_new_tokens + input_ids_seq_length
- if not has_default_max_length:
- logger.warn(
- f"Both `max_new_tokens` (={generation_config.max_new_tokens}) and `max_length`(="
- f"{generation_config.max_length}) seem to have been set. `max_new_tokens` will take precedence. "
- "Please refer to the documentation for more information. "
- "(https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)",
- UserWarning,
- )
-
- if input_ids_seq_length >= generation_config.max_length:
- input_ids_string = "decoder_input_ids" if self.config.is_encoder_decoder else "input_ids"
- logger.warning(
- f"Input length of {input_ids_string} is {input_ids_seq_length}, but `max_length` is set to"
- f" {generation_config.max_length}. This can lead to unexpected behavior. You should consider"
- " increasing `max_new_tokens`."
- )
-
- # 2. Set generation parameters if not already defined
- logits_processor = logits_processor if logits_processor is not None else LogitsProcessorList()
- stopping_criteria = stopping_criteria if stopping_criteria is not None else StoppingCriteriaList()
-
- logits_processor = self._get_logits_processor(
- generation_config=generation_config,
- input_ids_seq_length=input_ids_seq_length,
- encoder_input_ids=input_ids,
- prefix_allowed_tokens_fn=prefix_allowed_tokens_fn,
- logits_processor=logits_processor,
- )
-
- stopping_criteria = self._get_stopping_criteria(
- generation_config=generation_config, stopping_criteria=stopping_criteria
- )
- logits_warper = self._get_logits_warper(generation_config)
-
- unfinished_sequences = input_ids.new(input_ids.shape[0]).fill_(1)
- scores = None
- while True:
- model_inputs = self.prepare_inputs_for_generation(input_ids, **model_kwargs)
- # forward pass to get next token
- outputs = self(
- **model_inputs,
- return_dict=True,
- output_attentions=False,
- output_hidden_states=False,
- )
-
- next_token_logits = outputs.logits[:, -1, :]
-
- # pre-process distribution
- next_token_scores = logits_processor(input_ids, next_token_logits)
- next_token_scores = logits_warper(input_ids, next_token_scores)
-
- # sample
- probs = nn.functional.softmax(next_token_scores, dim=-1)
- if generation_config.do_sample:
- next_tokens = torch.multinomial(probs, num_samples=1).squeeze(1)
- else:
- next_tokens = torch.argmax(probs, dim=-1)
- # update generated ids, model inputs, and length for next step
- input_ids = torch.cat([input_ids, next_tokens[:, None]], dim=-1)
- model_kwargs = self._update_model_kwargs_for_generation(
- outputs, model_kwargs, is_encoder_decoder=self.config.is_encoder_decoder
- )
- unfinished_sequences = unfinished_sequences.mul(
- next_tokens.tile(eos_token_id_tensor.shape[0], 1).ne(eos_token_id_tensor.unsqueeze(1)).prod(dim=0)
- )
- if return_past_key_values:
- yield input_ids, outputs.past_key_values
- else:
- yield input_ids
- # stop when each sentence is finished, or if we exceed the maximum length
- if unfinished_sequences.max() == 0 or stopping_criteria(input_ids, scores):
- break
-
-
 class ChatGLMForSequenceClassification(ChatGLMPreTrainedModel):
  def __init__(self, config: ChatGLMConfig, empty_init=True, device=None):
  super().__init__(config)